{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use VPG to Play CartPole-v0\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras\n",
    "from tensorflow import nn\n",
    "from tensorflow import optimizers\n",
    "from tensorflow import losses\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "08:18:04 [INFO] env: <CartPoleEnv<CartPole-v0>>\n",
      "08:18:04 [INFO] action_space: Discrete(2)\n",
      "08:18:04 [INFO] observation_space: Box(-3.4028234663852886e+38, 3.4028234663852886e+38, (4,), float32)\n",
      "08:18:04 [INFO] reward_range: (-inf, inf)\n",
      "08:18:04 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}\n",
      "08:18:04 [INFO] _max_episode_steps: 200\n",
      "08:18:04 [INFO] _elapsed_steps: None\n",
      "08:18:04 [INFO] id: CartPole-v0\n",
      "08:18:04 [INFO] entry_point: gym.envs.classic_control:CartPoleEnv\n",
      "08:18:04 [INFO] reward_threshold: 195.0\n",
      "08:18:04 [INFO] nondeterministic: False\n",
      "08:18:04 [INFO] max_episode_steps: 200\n",
      "08:18:04 [INFO] _kwargs: {}\n",
      "08:18:04 [INFO] _env_name: CartPole\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VPGAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.policy_net = self.build_net(hidden_sizes=[],\n",
    "                output_size=self.action_n,\n",
    "                output_activation=nn.softmax,\n",
    "                loss=losses.categorical_crossentropy)\n",
    "\n",
    "    def build_net(self, hidden_sizes, output_size,\n",
    "            activation=nn.relu, output_activation=None,\n",
    "            use_bias=False, loss=losses.mse, learning_rate=0.005):\n",
    "        model = keras.Sequential()\n",
    "        for hidden_size in hidden_sizes:\n",
    "            model.add(layers.Dense(units=hidden_size,\n",
    "                    activation=activation, use_bias=use_bias))\n",
    "        model.add(layers.Dense(units=output_size,\n",
    "                activation=output_activation, use_bias=use_bias))\n",
    "        optimizer = optimizers.Adam(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        probs = self.policy_net.predict(observation[np.newaxis])[0]\n",
    "        action = np.random.choice(self.action_n, p=probs)\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        if self.mode == 'train':\n",
    "            self.learn()\n",
    "\n",
    "    def learn(self):\n",
    "        df = pd.DataFrame(np.array(self.trajectory, dtype=object).reshape(-1, 4),\n",
    "                columns=['state', 'reward', 'done', 'action'])\n",
    "        df['discount'] = self.gamma ** df.index.to_series()\n",
    "        df['discounted_reward'] = df['discount'] * df['reward']\n",
    "        df['discounted_return'] = df['discounted_reward'][::-1].cumsum()\n",
    "        states = np.stack(df['state'])\n",
    "        actions = np.eye(self.action_n)[df['action'].astype(int)]\n",
    "        sample_weight = df[['discounted_return',]].values.astype(float)\n",
    "        self.policy_net.fit(states, actions, sample_weight=sample_weight, verbose=0)\n",
    "\n",
    "\n",
    "agent = VPGAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "08:18:09 [INFO] ==== train ====\n",
      "08:18:12 [DEBUG] train episode 0: reward = 29.00, steps = 29\n",
      "08:18:12 [INFO] NumExpr defaulting to 8 threads.\n",
      "08:18:17 [DEBUG] train episode 1: reward = 39.00, steps = 39\n",
      "08:18:22 [DEBUG] train episode 2: reward = 44.00, steps = 44\n",
      "08:18:26 [DEBUG] train episode 3: reward = 38.00, steps = 38\n",
      "08:18:29 [DEBUG] train episode 4: reward = 33.00, steps = 33\n",
      "08:18:31 [DEBUG] train episode 5: reward = 20.00, steps = 20\n",
      "08:18:36 [DEBUG] train episode 6: reward = 49.00, steps = 49\n",
      "08:18:41 [DEBUG] train episode 7: reward = 56.00, steps = 56\n",
      "08:18:45 [DEBUG] train episode 8: reward = 30.00, steps = 30\n",
      "08:18:49 [DEBUG] train episode 9: reward = 44.00, steps = 44\n",
      "08:18:52 [DEBUG] train episode 10: reward = 26.00, steps = 26\n",
      "08:18:58 [DEBUG] train episode 11: reward = 64.00, steps = 64\n",
      "08:19:03 [DEBUG] train episode 12: reward = 56.00, steps = 56\n",
      "08:19:05 [DEBUG] train episode 13: reward = 24.00, steps = 24\n",
      "08:19:11 [DEBUG] train episode 14: reward = 65.00, steps = 65\n",
      "08:19:13 [DEBUG] train episode 15: reward = 14.00, steps = 14\n",
      "08:19:17 [DEBUG] train episode 16: reward = 44.00, steps = 44\n",
      "08:19:19 [DEBUG] train episode 17: reward = 20.00, steps = 20\n",
      "08:19:23 [DEBUG] train episode 18: reward = 39.00, steps = 39\n",
      "08:19:27 [DEBUG] train episode 19: reward = 47.00, steps = 47\n",
      "08:19:33 [DEBUG] train episode 20: reward = 61.00, steps = 61\n",
      "08:19:37 [DEBUG] train episode 21: reward = 38.00, steps = 38\n",
      "08:19:38 [DEBUG] train episode 22: reward = 19.00, steps = 19\n",
      "08:19:42 [DEBUG] train episode 23: reward = 40.00, steps = 40\n",
      "08:19:46 [DEBUG] train episode 24: reward = 39.00, steps = 39\n",
      "08:19:52 [DEBUG] train episode 25: reward = 64.00, steps = 64\n",
      "08:19:56 [DEBUG] train episode 26: reward = 47.00, steps = 47\n",
      "08:19:58 [DEBUG] train episode 27: reward = 15.00, steps = 15\n",
      "08:20:07 [DEBUG] train episode 28: reward = 94.00, steps = 94\n",
      "08:20:08 [DEBUG] train episode 29: reward = 19.00, steps = 19\n",
      "08:20:11 [DEBUG] train episode 30: reward = 22.00, steps = 22\n",
      "08:20:16 [DEBUG] train episode 31: reward = 57.00, steps = 57\n",
      "08:20:21 [DEBUG] train episode 32: reward = 50.00, steps = 50\n",
      "08:20:24 [DEBUG] train episode 33: reward = 31.00, steps = 31\n",
      "08:20:26 [DEBUG] train episode 34: reward = 23.00, steps = 23\n",
      "08:20:29 [DEBUG] train episode 35: reward = 26.00, steps = 26\n",
      "08:20:36 [DEBUG] train episode 36: reward = 73.00, steps = 73\n",
      "08:20:38 [DEBUG] train episode 37: reward = 25.00, steps = 25\n",
      "08:20:44 [DEBUG] train episode 38: reward = 67.00, steps = 67\n",
      "08:20:49 [DEBUG] train episode 39: reward = 46.00, steps = 46\n",
      "08:20:54 [DEBUG] train episode 40: reward = 54.00, steps = 54\n",
      "08:21:00 [DEBUG] train episode 41: reward = 66.00, steps = 66\n",
      "08:21:03 [DEBUG] train episode 42: reward = 30.00, steps = 30\n",
      "08:21:08 [DEBUG] train episode 43: reward = 52.00, steps = 52\n",
      "08:21:10 [DEBUG] train episode 44: reward = 26.00, steps = 26\n",
      "08:21:12 [DEBUG] train episode 45: reward = 22.00, steps = 22\n",
      "08:21:15 [DEBUG] train episode 46: reward = 27.00, steps = 27\n",
      "08:21:17 [DEBUG] train episode 47: reward = 16.00, steps = 16\n",
      "08:21:19 [DEBUG] train episode 48: reward = 18.00, steps = 18\n",
      "08:21:21 [DEBUG] train episode 49: reward = 22.00, steps = 22\n",
      "08:21:28 [DEBUG] train episode 50: reward = 76.00, steps = 76\n",
      "08:21:33 [DEBUG] train episode 51: reward = 58.00, steps = 58\n",
      "08:21:37 [DEBUG] train episode 52: reward = 41.00, steps = 41\n",
      "08:21:42 [DEBUG] train episode 53: reward = 47.00, steps = 47\n",
      "08:21:46 [DEBUG] train episode 54: reward = 41.00, steps = 41\n",
      "08:21:52 [DEBUG] train episode 55: reward = 63.00, steps = 63\n",
      "08:21:54 [DEBUG] train episode 56: reward = 26.00, steps = 26\n",
      "08:22:03 [DEBUG] train episode 57: reward = 93.00, steps = 93\n",
      "08:22:07 [DEBUG] train episode 58: reward = 43.00, steps = 43\n",
      "08:22:15 [DEBUG] train episode 59: reward = 83.00, steps = 83\n",
      "08:22:16 [DEBUG] train episode 60: reward = 14.00, steps = 14\n",
      "08:22:19 [DEBUG] train episode 61: reward = 29.00, steps = 29\n",
      "08:22:22 [DEBUG] train episode 62: reward = 37.00, steps = 37\n",
      "08:22:29 [DEBUG] train episode 63: reward = 76.00, steps = 76\n",
      "08:22:32 [DEBUG] train episode 64: reward = 22.00, steps = 22\n",
      "08:22:39 [DEBUG] train episode 65: reward = 78.00, steps = 78\n",
      "08:22:44 [DEBUG] train episode 66: reward = 54.00, steps = 54\n",
      "08:22:48 [DEBUG] train episode 67: reward = 45.00, steps = 45\n",
      "08:22:50 [DEBUG] train episode 68: reward = 13.00, steps = 13\n",
      "08:22:56 [DEBUG] train episode 69: reward = 68.00, steps = 68\n",
      "08:23:01 [DEBUG] train episode 70: reward = 60.00, steps = 60\n",
      "08:23:08 [DEBUG] train episode 71: reward = 76.00, steps = 76\n",
      "08:23:17 [DEBUG] train episode 72: reward = 86.00, steps = 86\n",
      "08:23:22 [DEBUG] train episode 73: reward = 65.00, steps = 65\n",
      "08:23:24 [DEBUG] train episode 74: reward = 18.00, steps = 18\n",
      "08:23:30 [DEBUG] train episode 75: reward = 57.00, steps = 57\n",
      "08:23:32 [DEBUG] train episode 76: reward = 26.00, steps = 26\n",
      "08:23:34 [DEBUG] train episode 77: reward = 22.00, steps = 22\n",
      "08:23:41 [DEBUG] train episode 78: reward = 71.00, steps = 71\n",
      "08:23:49 [DEBUG] train episode 79: reward = 90.00, steps = 90\n",
      "08:23:53 [DEBUG] train episode 80: reward = 45.00, steps = 45\n",
      "08:23:58 [DEBUG] train episode 81: reward = 50.00, steps = 50\n",
      "08:24:01 [DEBUG] train episode 82: reward = 26.00, steps = 26\n",
      "08:24:02 [DEBUG] train episode 83: reward = 14.00, steps = 14\n",
      "08:24:06 [DEBUG] train episode 84: reward = 46.00, steps = 46\n",
      "08:24:11 [DEBUG] train episode 85: reward = 50.00, steps = 50\n",
      "08:24:14 [DEBUG] train episode 86: reward = 23.00, steps = 23\n",
      "08:24:18 [DEBUG] train episode 87: reward = 46.00, steps = 46\n",
      "08:24:21 [DEBUG] train episode 88: reward = 37.00, steps = 37\n",
      "08:24:24 [DEBUG] train episode 89: reward = 22.00, steps = 22\n",
      "08:24:31 [DEBUG] train episode 90: reward = 80.00, steps = 80\n",
      "08:24:36 [DEBUG] train episode 91: reward = 52.00, steps = 52\n",
      "08:24:45 [DEBUG] train episode 92: reward = 98.00, steps = 98\n",
      "08:24:48 [DEBUG] train episode 93: reward = 35.00, steps = 35\n",
      "08:24:51 [DEBUG] train episode 94: reward = 24.00, steps = 24\n",
      "08:24:59 [DEBUG] train episode 95: reward = 95.00, steps = 95\n",
      "08:25:03 [DEBUG] train episode 96: reward = 38.00, steps = 38\n",
      "08:25:08 [DEBUG] train episode 97: reward = 55.00, steps = 55\n",
      "08:25:12 [DEBUG] train episode 98: reward = 42.00, steps = 42\n",
      "08:25:17 [DEBUG] train episode 99: reward = 50.00, steps = 50\n",
      "08:25:25 [DEBUG] train episode 100: reward = 86.00, steps = 86\n",
      "08:25:29 [DEBUG] train episode 101: reward = 45.00, steps = 45\n",
      "08:25:32 [DEBUG] train episode 102: reward = 31.00, steps = 31\n",
      "08:25:35 [DEBUG] train episode 103: reward = 30.00, steps = 30\n",
      "08:25:37 [DEBUG] train episode 104: reward = 19.00, steps = 19\n",
      "08:25:41 [DEBUG] train episode 105: reward = 34.00, steps = 34\n",
      "08:25:46 [DEBUG] train episode 106: reward = 57.00, steps = 57\n",
      "08:25:48 [DEBUG] train episode 107: reward = 19.00, steps = 19\n",
      "08:25:53 [DEBUG] train episode 108: reward = 52.00, steps = 52\n",
      "08:25:56 [DEBUG] train episode 109: reward = 35.00, steps = 35\n",
      "08:25:58 [DEBUG] train episode 110: reward = 18.00, steps = 18\n",
      "08:26:09 [DEBUG] train episode 111: reward = 118.00, steps = 118\n",
      "08:26:13 [DEBUG] train episode 112: reward = 45.00, steps = 45\n",
      "08:26:16 [DEBUG] train episode 113: reward = 35.00, steps = 35\n",
      "08:26:23 [DEBUG] train episode 114: reward = 69.00, steps = 69\n",
      "08:26:26 [DEBUG] train episode 115: reward = 39.00, steps = 39\n",
      "08:26:37 [DEBUG] train episode 116: reward = 114.00, steps = 114\n",
      "08:26:39 [DEBUG] train episode 117: reward = 18.00, steps = 18\n",
      "08:26:44 [DEBUG] train episode 118: reward = 55.00, steps = 55\n",
      "08:26:47 [DEBUG] train episode 119: reward = 29.00, steps = 29\n",
      "08:26:48 [DEBUG] train episode 120: reward = 15.00, steps = 15\n",
      "08:26:52 [DEBUG] train episode 121: reward = 45.00, steps = 45\n",
      "08:26:54 [DEBUG] train episode 122: reward = 21.00, steps = 21\n",
      "08:27:00 [DEBUG] train episode 123: reward = 57.00, steps = 57\n",
      "08:27:04 [DEBUG] train episode 124: reward = 44.00, steps = 44\n",
      "08:27:10 [DEBUG] train episode 125: reward = 71.00, steps = 71\n",
      "08:27:14 [DEBUG] train episode 126: reward = 38.00, steps = 38\n",
      "08:27:20 [DEBUG] train episode 127: reward = 65.00, steps = 65\n",
      "08:27:23 [DEBUG] train episode 128: reward = 34.00, steps = 34\n",
      "08:27:26 [DEBUG] train episode 129: reward = 29.00, steps = 29\n",
      "08:27:30 [DEBUG] train episode 130: reward = 36.00, steps = 36\n",
      "08:27:34 [DEBUG] train episode 131: reward = 50.00, steps = 50\n",
      "08:27:39 [DEBUG] train episode 132: reward = 53.00, steps = 53\n",
      "08:27:44 [DEBUG] train episode 133: reward = 54.00, steps = 54\n",
      "08:27:47 [DEBUG] train episode 134: reward = 30.00, steps = 30\n",
      "08:27:55 [DEBUG] train episode 135: reward = 80.00, steps = 80\n",
      "08:28:05 [DEBUG] train episode 136: reward = 113.00, steps = 113\n",
      "08:28:15 [DEBUG] train episode 137: reward = 108.00, steps = 108\n",
      "08:28:20 [DEBUG] train episode 138: reward = 60.00, steps = 60\n",
      "08:28:25 [DEBUG] train episode 139: reward = 54.00, steps = 54\n",
      "08:28:29 [DEBUG] train episode 140: reward = 36.00, steps = 36\n",
      "08:28:32 [DEBUG] train episode 141: reward = 35.00, steps = 35\n",
      "08:28:36 [DEBUG] train episode 142: reward = 42.00, steps = 42\n",
      "08:28:37 [DEBUG] train episode 143: reward = 11.00, steps = 11\n",
      "08:28:40 [DEBUG] train episode 144: reward = 30.00, steps = 30\n",
      "08:28:49 [DEBUG] train episode 145: reward = 94.00, steps = 94\n",
      "08:28:50 [DEBUG] train episode 146: reward = 15.00, steps = 15\n",
      "08:28:54 [DEBUG] train episode 147: reward = 40.00, steps = 40\n",
      "08:28:57 [DEBUG] train episode 148: reward = 30.00, steps = 30\n",
      "08:29:02 [DEBUG] train episode 149: reward = 60.00, steps = 60\n",
      "08:29:05 [DEBUG] train episode 150: reward = 30.00, steps = 30\n",
      "08:29:08 [DEBUG] train episode 151: reward = 30.00, steps = 30\n",
      "08:29:10 [DEBUG] train episode 152: reward = 19.00, steps = 19\n",
      "08:29:18 [DEBUG] train episode 153: reward = 88.00, steps = 88\n",
      "08:29:24 [DEBUG] train episode 154: reward = 67.00, steps = 67\n",
      "08:29:33 [DEBUG] train episode 155: reward = 101.00, steps = 101\n",
      "08:29:36 [DEBUG] train episode 156: reward = 21.00, steps = 21\n",
      "08:29:47 [DEBUG] train episode 157: reward = 129.00, steps = 129\n",
      "08:29:55 [DEBUG] train episode 158: reward = 80.00, steps = 80\n",
      "08:29:59 [DEBUG] train episode 159: reward = 43.00, steps = 43\n",
      "08:30:06 [DEBUG] train episode 160: reward = 73.00, steps = 73\n",
      "08:30:09 [DEBUG] train episode 161: reward = 40.00, steps = 40\n",
      "08:30:15 [DEBUG] train episode 162: reward = 58.00, steps = 58\n",
      "08:30:18 [DEBUG] train episode 163: reward = 34.00, steps = 34\n",
      "08:30:22 [DEBUG] train episode 164: reward = 47.00, steps = 47\n",
      "08:30:28 [DEBUG] train episode 165: reward = 63.00, steps = 63\n",
      "08:30:40 [DEBUG] train episode 166: reward = 125.00, steps = 125\n",
      "08:30:45 [DEBUG] train episode 167: reward = 53.00, steps = 53\n",
      "08:30:51 [DEBUG] train episode 168: reward = 66.00, steps = 66\n",
      "08:30:57 [DEBUG] train episode 169: reward = 72.00, steps = 72\n",
      "08:31:12 [DEBUG] train episode 170: reward = 155.00, steps = 155\n",
      "08:31:14 [DEBUG] train episode 171: reward = 26.00, steps = 26\n",
      "08:31:18 [DEBUG] train episode 172: reward = 46.00, steps = 46\n",
      "08:31:23 [DEBUG] train episode 173: reward = 45.00, steps = 45\n",
      "08:31:33 [DEBUG] train episode 174: reward = 112.00, steps = 112\n",
      "08:31:45 [DEBUG] train episode 175: reward = 119.00, steps = 119\n",
      "08:31:53 [DEBUG] train episode 176: reward = 84.00, steps = 84\n",
      "08:32:00 [DEBUG] train episode 177: reward = 82.00, steps = 82\n",
      "08:32:06 [DEBUG] train episode 178: reward = 55.00, steps = 55\n",
      "08:32:12 [DEBUG] train episode 179: reward = 63.00, steps = 63\n",
      "08:32:15 [DEBUG] train episode 180: reward = 34.00, steps = 34\n",
      "08:32:20 [DEBUG] train episode 181: reward = 47.00, steps = 47\n",
      "08:32:23 [DEBUG] train episode 182: reward = 28.00, steps = 28\n",
      "08:32:30 [DEBUG] train episode 183: reward = 70.00, steps = 70\n",
      "08:32:33 [DEBUG] train episode 184: reward = 31.00, steps = 31\n",
      "08:32:37 [DEBUG] train episode 185: reward = 53.00, steps = 53\n",
      "08:32:42 [DEBUG] train episode 186: reward = 49.00, steps = 49\n",
      "08:32:56 [DEBUG] train episode 187: reward = 152.00, steps = 152\n",
      "08:33:00 [DEBUG] train episode 188: reward = 49.00, steps = 49\n",
      "08:33:05 [DEBUG] train episode 189: reward = 53.00, steps = 53\n",
      "08:33:14 [DEBUG] train episode 190: reward = 101.00, steps = 101\n",
      "08:33:21 [DEBUG] train episode 191: reward = 75.00, steps = 75\n",
      "08:33:31 [DEBUG] train episode 192: reward = 111.00, steps = 111\n",
      "08:33:36 [DEBUG] train episode 193: reward = 46.00, steps = 46\n",
      "08:33:42 [DEBUG] train episode 194: reward = 71.00, steps = 71\n",
      "08:33:48 [DEBUG] train episode 195: reward = 58.00, steps = 58\n",
      "08:33:50 [DEBUG] train episode 196: reward = 20.00, steps = 20\n",
      "08:33:55 [DEBUG] train episode 197: reward = 63.00, steps = 63\n",
      "08:34:06 [DEBUG] train episode 198: reward = 117.00, steps = 117\n",
      "08:34:12 [DEBUG] train episode 199: reward = 69.00, steps = 69\n",
      "08:34:20 [DEBUG] train episode 200: reward = 78.00, steps = 78\n",
      "08:34:22 [DEBUG] train episode 201: reward = 28.00, steps = 28\n",
      "08:34:31 [DEBUG] train episode 202: reward = 93.00, steps = 93\n",
      "08:34:43 [DEBUG] train episode 203: reward = 134.00, steps = 134\n",
      "08:34:46 [DEBUG] train episode 204: reward = 29.00, steps = 29\n",
      "08:34:59 [DEBUG] train episode 205: reward = 143.00, steps = 143\n",
      "08:35:02 [DEBUG] train episode 206: reward = 41.00, steps = 41\n",
      "08:35:09 [DEBUG] train episode 207: reward = 71.00, steps = 71\n",
      "08:35:13 [DEBUG] train episode 208: reward = 38.00, steps = 38\n",
      "08:35:16 [DEBUG] train episode 209: reward = 40.00, steps = 40\n",
      "08:35:26 [DEBUG] train episode 210: reward = 104.00, steps = 104\n",
      "08:35:29 [DEBUG] train episode 211: reward = 37.00, steps = 37\n",
      "08:35:35 [DEBUG] train episode 212: reward = 59.00, steps = 59\n",
      "08:35:37 [DEBUG] train episode 213: reward = 25.00, steps = 25\n",
      "08:35:40 [DEBUG] train episode 214: reward = 32.00, steps = 32\n",
      "08:35:56 [DEBUG] train episode 215: reward = 177.00, steps = 177\n",
      "08:36:03 [DEBUG] train episode 216: reward = 77.00, steps = 77\n",
      "08:36:06 [DEBUG] train episode 217: reward = 29.00, steps = 29\n",
      "08:36:08 [DEBUG] train episode 218: reward = 21.00, steps = 21\n",
      "08:36:21 [DEBUG] train episode 219: reward = 140.00, steps = 140\n",
      "08:36:24 [DEBUG] train episode 220: reward = 32.00, steps = 32\n",
      "08:36:27 [DEBUG] train episode 221: reward = 29.00, steps = 29\n",
      "08:36:32 [DEBUG] train episode 222: reward = 58.00, steps = 58\n",
      "08:36:39 [DEBUG] train episode 223: reward = 74.00, steps = 74\n",
      "08:36:47 [DEBUG] train episode 224: reward = 87.00, steps = 87\n",
      "08:36:55 [DEBUG] train episode 225: reward = 86.00, steps = 86\n",
      "08:37:10 [DEBUG] train episode 226: reward = 175.00, steps = 175\n",
      "08:37:27 [DEBUG] train episode 227: reward = 188.00, steps = 188\n",
      "08:37:36 [DEBUG] train episode 228: reward = 91.00, steps = 91\n",
      "08:37:43 [DEBUG] train episode 229: reward = 74.00, steps = 74\n",
      "08:37:52 [DEBUG] train episode 230: reward = 103.00, steps = 103\n",
      "08:38:02 [DEBUG] train episode 231: reward = 106.00, steps = 106\n",
      "08:38:20 [DEBUG] train episode 232: reward = 200.00, steps = 200\n",
      "08:38:25 [DEBUG] train episode 233: reward = 64.00, steps = 64\n",
      "08:38:33 [DEBUG] train episode 234: reward = 82.00, steps = 82\n",
      "08:38:38 [DEBUG] train episode 235: reward = 60.00, steps = 60\n",
      "08:38:44 [DEBUG] train episode 236: reward = 68.00, steps = 68\n",
      "08:39:01 [DEBUG] train episode 237: reward = 179.00, steps = 179\n",
      "08:39:07 [DEBUG] train episode 238: reward = 67.00, steps = 67\n",
      "08:39:10 [DEBUG] train episode 239: reward = 28.00, steps = 28\n",
      "08:39:15 [DEBUG] train episode 240: reward = 62.00, steps = 62\n",
      "08:39:18 [DEBUG] train episode 241: reward = 24.00, steps = 24\n",
      "08:39:36 [DEBUG] train episode 242: reward = 200.00, steps = 200\n",
      "08:39:54 [DEBUG] train episode 243: reward = 200.00, steps = 200\n",
      "08:40:04 [DEBUG] train episode 244: reward = 107.00, steps = 107\n",
      "08:40:12 [DEBUG] train episode 245: reward = 90.00, steps = 90\n",
      "08:40:21 [DEBUG] train episode 246: reward = 89.00, steps = 89\n",
      "08:40:30 [DEBUG] train episode 247: reward = 98.00, steps = 98\n",
      "08:40:45 [DEBUG] train episode 248: reward = 168.00, steps = 168\n",
      "08:40:55 [DEBUG] train episode 249: reward = 108.00, steps = 108\n",
      "08:41:07 [DEBUG] train episode 250: reward = 134.00, steps = 134\n",
      "08:41:16 [DEBUG] train episode 251: reward = 103.00, steps = 103\n",
      "08:41:22 [DEBUG] train episode 252: reward = 59.00, steps = 59\n",
      "08:41:27 [DEBUG] train episode 253: reward = 60.00, steps = 60\n",
      "08:41:38 [DEBUG] train episode 254: reward = 110.00, steps = 110\n",
      "08:41:52 [DEBUG] train episode 255: reward = 162.00, steps = 162\n",
      "08:42:07 [DEBUG] train episode 256: reward = 166.00, steps = 166\n",
      "08:42:10 [DEBUG] train episode 257: reward = 35.00, steps = 35\n",
      "08:42:21 [DEBUG] train episode 258: reward = 118.00, steps = 118\n",
      "08:42:24 [DEBUG] train episode 259: reward = 34.00, steps = 34\n",
      "08:42:30 [DEBUG] train episode 260: reward = 60.00, steps = 60\n",
      "08:42:40 [DEBUG] train episode 261: reward = 110.00, steps = 110\n",
      "08:42:48 [DEBUG] train episode 262: reward = 88.00, steps = 88\n",
      "08:42:56 [DEBUG] train episode 263: reward = 85.00, steps = 85\n",
      "08:43:02 [DEBUG] train episode 264: reward = 68.00, steps = 68\n",
      "08:43:11 [DEBUG] train episode 265: reward = 94.00, steps = 94\n",
      "08:43:12 [DEBUG] train episode 266: reward = 16.00, steps = 16\n",
      "08:43:15 [DEBUG] train episode 267: reward = 32.00, steps = 32\n",
      "08:43:19 [DEBUG] train episode 268: reward = 39.00, steps = 39\n",
      "08:43:37 [DEBUG] train episode 269: reward = 196.00, steps = 196\n",
      "08:43:39 [DEBUG] train episode 270: reward = 22.00, steps = 22\n",
      "08:43:43 [DEBUG] train episode 271: reward = 44.00, steps = 44\n",
      "08:44:01 [DEBUG] train episode 272: reward = 200.00, steps = 200\n",
      "08:44:07 [DEBUG] train episode 273: reward = 59.00, steps = 59\n",
      "08:44:17 [DEBUG] train episode 274: reward = 116.00, steps = 116\n",
      "08:44:28 [DEBUG] train episode 275: reward = 119.00, steps = 119\n",
      "08:44:40 [DEBUG] train episode 276: reward = 126.00, steps = 126\n",
      "08:44:55 [DEBUG] train episode 277: reward = 174.00, steps = 174\n",
      "08:45:13 [DEBUG] train episode 278: reward = 200.00, steps = 200\n",
      "08:45:29 [DEBUG] train episode 279: reward = 175.00, steps = 175\n",
      "08:45:36 [DEBUG] train episode 280: reward = 78.00, steps = 78\n",
      "08:45:49 [DEBUG] train episode 281: reward = 136.00, steps = 136\n",
      "08:46:03 [DEBUG] train episode 282: reward = 139.00, steps = 139\n",
      "08:46:04 [DEBUG] train episode 283: reward = 15.00, steps = 15\n",
      "08:46:13 [DEBUG] train episode 284: reward = 88.00, steps = 88\n",
      "08:46:18 [DEBUG] train episode 285: reward = 52.00, steps = 52\n",
      "08:46:20 [DEBUG] train episode 286: reward = 22.00, steps = 22\n",
      "08:46:39 [DEBUG] train episode 287: reward = 200.00, steps = 200\n",
      "08:46:43 [DEBUG] train episode 288: reward = 27.00, steps = 27\n",
      "08:46:59 [DEBUG] train episode 289: reward = 169.00, steps = 169\n",
      "08:47:18 [DEBUG] train episode 290: reward = 200.00, steps = 200\n",
      "08:47:28 [DEBUG] train episode 291: reward = 101.00, steps = 101\n",
      "08:47:47 [DEBUG] train episode 292: reward = 200.00, steps = 200\n",
      "08:47:51 [DEBUG] train episode 293: reward = 37.00, steps = 37\n",
      "08:48:05 [DEBUG] train episode 294: reward = 146.00, steps = 146\n",
      "08:48:10 [DEBUG] train episode 295: reward = 45.00, steps = 45\n",
      "08:48:29 [DEBUG] train episode 296: reward = 200.00, steps = 200\n",
      "08:48:40 [DEBUG] train episode 297: reward = 111.00, steps = 111\n",
      "08:48:52 [DEBUG] train episode 298: reward = 125.00, steps = 125\n",
      "08:49:06 [DEBUG] train episode 299: reward = 145.00, steps = 145\n",
      "08:49:20 [DEBUG] train episode 300: reward = 139.00, steps = 139\n",
      "08:49:37 [DEBUG] train episode 301: reward = 182.00, steps = 182\n",
      "08:49:56 [DEBUG] train episode 302: reward = 200.00, steps = 200\n",
      "08:50:16 [DEBUG] train episode 303: reward = 199.00, steps = 199\n",
      "08:50:27 [DEBUG] train episode 304: reward = 112.00, steps = 112\n",
      "08:50:31 [DEBUG] train episode 305: reward = 45.00, steps = 45\n",
      "08:50:44 [DEBUG] train episode 306: reward = 128.00, steps = 128\n",
      "08:50:47 [DEBUG] train episode 307: reward = 30.00, steps = 30\n",
      "08:51:04 [DEBUG] train episode 308: reward = 183.00, steps = 183\n",
      "08:51:07 [DEBUG] train episode 309: reward = 27.00, steps = 27\n",
      "08:51:22 [DEBUG] train episode 310: reward = 147.00, steps = 147\n",
      "08:51:39 [DEBUG] train episode 311: reward = 200.00, steps = 200\n",
      "08:51:57 [DEBUG] train episode 312: reward = 200.00, steps = 200\n",
      "08:52:15 [DEBUG] train episode 313: reward = 200.00, steps = 200\n",
      "08:52:30 [DEBUG] train episode 314: reward = 175.00, steps = 175\n",
      "08:52:38 [DEBUG] train episode 315: reward = 90.00, steps = 90\n",
      "08:52:45 [DEBUG] train episode 316: reward = 82.00, steps = 82\n",
      "08:52:58 [DEBUG] train episode 317: reward = 141.00, steps = 141\n",
      "08:53:15 [DEBUG] train episode 318: reward = 187.00, steps = 187\n",
      "08:53:19 [DEBUG] train episode 319: reward = 49.00, steps = 49\n",
      "08:53:37 [DEBUG] train episode 320: reward = 200.00, steps = 200\n",
      "08:53:53 [DEBUG] train episode 321: reward = 178.00, steps = 178\n",
      "08:54:04 [DEBUG] train episode 322: reward = 125.00, steps = 125\n",
      "08:54:20 [DEBUG] train episode 323: reward = 177.00, steps = 177\n",
      "08:54:23 [DEBUG] train episode 324: reward = 37.00, steps = 37\n",
      "08:54:26 [DEBUG] train episode 325: reward = 28.00, steps = 28\n",
      "08:54:35 [DEBUG] train episode 326: reward = 96.00, steps = 96\n",
      "08:54:36 [DEBUG] train episode 327: reward = 16.00, steps = 16\n",
      "08:54:41 [DEBUG] train episode 328: reward = 56.00, steps = 56\n",
      "08:54:44 [DEBUG] train episode 329: reward = 24.00, steps = 24\n",
      "08:54:56 [DEBUG] train episode 330: reward = 143.00, steps = 143\n",
      "08:55:07 [DEBUG] train episode 331: reward = 121.00, steps = 121\n",
      "08:55:25 [DEBUG] train episode 332: reward = 200.00, steps = 200\n",
      "08:55:42 [DEBUG] train episode 333: reward = 185.00, steps = 185\n",
      "08:55:55 [DEBUG] train episode 334: reward = 145.00, steps = 145\n",
      "08:55:59 [DEBUG] train episode 335: reward = 47.00, steps = 47\n",
      "08:56:12 [DEBUG] train episode 336: reward = 148.00, steps = 148\n",
      "08:56:30 [DEBUG] train episode 337: reward = 200.00, steps = 200\n",
      "08:56:41 [DEBUG] train episode 338: reward = 123.00, steps = 123\n",
      "08:56:46 [DEBUG] train episode 339: reward = 55.00, steps = 55\n",
      "08:56:57 [DEBUG] train episode 340: reward = 126.00, steps = 126\n",
      "08:57:13 [DEBUG] train episode 341: reward = 179.00, steps = 179\n",
      "08:57:30 [DEBUG] train episode 342: reward = 197.00, steps = 197\n",
      "08:57:33 [DEBUG] train episode 343: reward = 29.00, steps = 29\n",
      "08:57:47 [DEBUG] train episode 344: reward = 152.00, steps = 152\n",
      "08:58:00 [DEBUG] train episode 345: reward = 144.00, steps = 144\n",
      "08:58:04 [DEBUG] train episode 346: reward = 45.00, steps = 45\n",
      "08:58:14 [DEBUG] train episode 347: reward = 116.00, steps = 116\n",
      "08:58:27 [DEBUG] train episode 348: reward = 141.00, steps = 141\n",
      "08:58:40 [DEBUG] train episode 349: reward = 149.00, steps = 149\n",
      "08:58:58 [DEBUG] train episode 350: reward = 200.00, steps = 200\n",
      "08:59:15 [DEBUG] train episode 351: reward = 200.00, steps = 200\n",
      "08:59:27 [DEBUG] train episode 352: reward = 125.00, steps = 125\n",
      "08:59:45 [DEBUG] train episode 353: reward = 200.00, steps = 200\n",
      "08:59:53 [DEBUG] train episode 354: reward = 97.00, steps = 97\n",
      "09:00:04 [DEBUG] train episode 355: reward = 118.00, steps = 118\n",
      "09:00:22 [DEBUG] train episode 356: reward = 200.00, steps = 200\n",
      "09:00:33 [DEBUG] train episode 357: reward = 125.00, steps = 125\n",
      "09:00:40 [DEBUG] train episode 358: reward = 75.00, steps = 75\n",
      "09:00:51 [DEBUG] train episode 359: reward = 126.00, steps = 126\n",
      "09:01:02 [DEBUG] train episode 360: reward = 127.00, steps = 127\n",
      "09:01:06 [DEBUG] train episode 361: reward = 35.00, steps = 35\n",
      "09:01:11 [DEBUG] train episode 362: reward = 59.00, steps = 59\n",
      "09:01:24 [DEBUG] train episode 363: reward = 145.00, steps = 145\n",
      "09:01:28 [DEBUG] train episode 364: reward = 44.00, steps = 44\n",
      "09:01:45 [DEBUG] train episode 365: reward = 194.00, steps = 194\n",
      "09:02:03 [DEBUG] train episode 366: reward = 200.00, steps = 200\n",
      "09:02:19 [DEBUG] train episode 367: reward = 174.00, steps = 174\n",
      "09:02:21 [DEBUG] train episode 368: reward = 28.00, steps = 28\n",
      "09:02:39 [DEBUG] train episode 369: reward = 199.00, steps = 199\n",
      "09:02:56 [DEBUG] train episode 370: reward = 194.00, steps = 194\n",
      "09:03:00 [DEBUG] train episode 371: reward = 37.00, steps = 37\n",
      "09:03:13 [DEBUG] train episode 372: reward = 151.00, steps = 151\n",
      "09:03:31 [DEBUG] train episode 373: reward = 200.00, steps = 200\n",
      "09:03:49 [DEBUG] train episode 374: reward = 200.00, steps = 200\n",
      "09:04:03 [DEBUG] train episode 375: reward = 165.00, steps = 165\n",
      "09:04:21 [DEBUG] train episode 376: reward = 200.00, steps = 200\n",
      "09:04:29 [DEBUG] train episode 377: reward = 89.00, steps = 89\n",
      "09:04:47 [DEBUG] train episode 378: reward = 200.00, steps = 200\n",
      "09:05:05 [DEBUG] train episode 379: reward = 200.00, steps = 200\n",
      "09:05:22 [DEBUG] train episode 380: reward = 200.00, steps = 200\n",
      "09:05:40 [DEBUG] train episode 381: reward = 200.00, steps = 200\n",
      "09:05:58 [DEBUG] train episode 382: reward = 200.00, steps = 200\n",
      "09:06:09 [DEBUG] train episode 383: reward = 125.00, steps = 125\n",
      "09:06:27 [DEBUG] train episode 384: reward = 200.00, steps = 200\n",
      "09:06:46 [DEBUG] train episode 385: reward = 200.00, steps = 200\n",
      "09:07:04 [DEBUG] train episode 386: reward = 195.00, steps = 195\n",
      "09:07:24 [DEBUG] train episode 387: reward = 200.00, steps = 200\n",
      "09:07:39 [DEBUG] train episode 388: reward = 154.00, steps = 154\n",
      "09:07:57 [DEBUG] train episode 389: reward = 192.00, steps = 192\n",
      "09:08:16 [DEBUG] train episode 390: reward = 200.00, steps = 200\n",
      "09:08:34 [DEBUG] train episode 391: reward = 200.00, steps = 200\n",
      "09:08:51 [DEBUG] train episode 392: reward = 183.00, steps = 183\n",
      "09:09:10 [DEBUG] train episode 393: reward = 197.00, steps = 197\n",
      "09:09:29 [DEBUG] train episode 394: reward = 200.00, steps = 200\n",
      "09:09:49 [DEBUG] train episode 395: reward = 200.00, steps = 200\n",
      "09:10:07 [DEBUG] train episode 396: reward = 200.00, steps = 200\n",
      "09:10:26 [DEBUG] train episode 397: reward = 200.00, steps = 200\n",
      "09:10:38 [DEBUG] train episode 398: reward = 130.00, steps = 130\n",
      "09:10:57 [DEBUG] train episode 399: reward = 200.00, steps = 200\n",
      "09:11:16 [DEBUG] train episode 400: reward = 200.00, steps = 200\n",
      "09:11:23 [DEBUG] train episode 401: reward = 74.00, steps = 74\n",
      "09:11:39 [DEBUG] train episode 402: reward = 163.00, steps = 163\n",
      "09:11:51 [DEBUG] train episode 403: reward = 135.00, steps = 135\n",
      "09:12:10 [DEBUG] train episode 404: reward = 200.00, steps = 200\n",
      "09:12:24 [DEBUG] train episode 405: reward = 154.00, steps = 154\n",
      "09:12:36 [DEBUG] train episode 406: reward = 124.00, steps = 124\n",
      "09:12:47 [DEBUG] train episode 407: reward = 114.00, steps = 114\n",
      "09:12:57 [DEBUG] train episode 408: reward = 111.00, steps = 111\n",
      "09:13:09 [DEBUG] train episode 409: reward = 117.00, steps = 117\n",
      "09:13:22 [DEBUG] train episode 410: reward = 137.00, steps = 137\n",
      "09:13:32 [DEBUG] train episode 411: reward = 105.00, steps = 105\n",
      "09:13:45 [DEBUG] train episode 412: reward = 137.00, steps = 137\n",
      "09:13:58 [DEBUG] train episode 413: reward = 151.00, steps = 151\n",
      "09:14:16 [DEBUG] train episode 414: reward = 200.00, steps = 200\n",
      "09:14:34 [DEBUG] train episode 415: reward = 200.00, steps = 200\n",
      "09:14:45 [DEBUG] train episode 416: reward = 118.00, steps = 118\n",
      "09:14:57 [DEBUG] train episode 417: reward = 126.00, steps = 126\n",
      "09:15:09 [DEBUG] train episode 418: reward = 136.00, steps = 136\n",
      "09:15:17 [DEBUG] train episode 419: reward = 86.00, steps = 86\n",
      "09:15:22 [DEBUG] train episode 420: reward = 56.00, steps = 56\n",
      "09:15:36 [DEBUG] train episode 421: reward = 150.00, steps = 150\n",
      "09:15:39 [DEBUG] train episode 422: reward = 34.00, steps = 34\n",
      "09:15:46 [DEBUG] train episode 423: reward = 66.00, steps = 66\n",
      "09:16:04 [DEBUG] train episode 424: reward = 200.00, steps = 200\n",
      "09:16:22 [DEBUG] train episode 425: reward = 196.00, steps = 196\n",
      "09:16:37 [DEBUG] train episode 426: reward = 145.00, steps = 145\n",
      "09:16:49 [DEBUG] train episode 427: reward = 131.00, steps = 131\n",
      "09:17:00 [DEBUG] train episode 428: reward = 108.00, steps = 108\n",
      "09:17:18 [DEBUG] train episode 429: reward = 188.00, steps = 188\n",
      "09:17:38 [DEBUG] train episode 430: reward = 200.00, steps = 200\n",
      "09:17:55 [DEBUG] train episode 431: reward = 163.00, steps = 163\n",
      "09:18:07 [DEBUG] train episode 432: reward = 117.00, steps = 117\n",
      "09:18:21 [DEBUG] train episode 433: reward = 151.00, steps = 151\n",
      "09:18:40 [DEBUG] train episode 434: reward = 200.00, steps = 200\n",
      "09:18:51 [DEBUG] train episode 435: reward = 123.00, steps = 123\n",
      "09:19:08 [DEBUG] train episode 436: reward = 199.00, steps = 199\n",
      "09:19:26 [DEBUG] train episode 437: reward = 200.00, steps = 200\n",
      "09:19:36 [DEBUG] train episode 438: reward = 114.00, steps = 114\n",
      "09:19:44 [DEBUG] train episode 439: reward = 90.00, steps = 90\n",
      "09:20:02 [DEBUG] train episode 440: reward = 198.00, steps = 198\n",
      "09:20:12 [DEBUG] train episode 441: reward = 114.00, steps = 114\n",
      "09:20:30 [DEBUG] train episode 442: reward = 200.00, steps = 200\n",
      "09:20:47 [DEBUG] train episode 443: reward = 200.00, steps = 200\n",
      "09:21:05 [DEBUG] train episode 444: reward = 200.00, steps = 200\n",
      "09:21:23 [DEBUG] train episode 445: reward = 200.00, steps = 200\n",
      "09:21:40 [DEBUG] train episode 446: reward = 200.00, steps = 200\n",
      "09:21:53 [DEBUG] train episode 447: reward = 137.00, steps = 137\n",
      "09:22:10 [DEBUG] train episode 448: reward = 200.00, steps = 200\n",
      "09:22:28 [DEBUG] train episode 449: reward = 200.00, steps = 200\n",
      "09:22:40 [DEBUG] train episode 450: reward = 130.00, steps = 130\n",
      "09:22:46 [DEBUG] train episode 451: reward = 57.00, steps = 57\n",
      "09:23:04 [DEBUG] train episode 452: reward = 200.00, steps = 200\n",
      "09:23:16 [DEBUG] train episode 453: reward = 135.00, steps = 135\n",
      "09:23:33 [DEBUG] train episode 454: reward = 200.00, steps = 200\n",
      "09:23:44 [DEBUG] train episode 455: reward = 124.00, steps = 124\n",
      "09:24:02 [DEBUG] train episode 456: reward = 200.00, steps = 200\n",
      "09:24:11 [DEBUG] train episode 457: reward = 110.00, steps = 110\n",
      "09:24:29 [DEBUG] train episode 458: reward = 200.00, steps = 200\n",
      "09:24:47 [DEBUG] train episode 459: reward = 197.00, steps = 197\n",
      "09:24:57 [DEBUG] train episode 460: reward = 110.00, steps = 110\n",
      "09:25:13 [DEBUG] train episode 461: reward = 189.00, steps = 189\n",
      "09:25:28 [DEBUG] train episode 462: reward = 170.00, steps = 170\n",
      "09:25:39 [DEBUG] train episode 463: reward = 118.00, steps = 118\n",
      "09:25:56 [DEBUG] train episode 464: reward = 196.00, steps = 196\n",
      "09:26:08 [DEBUG] train episode 465: reward = 127.00, steps = 127\n",
      "09:26:10 [DEBUG] train episode 466: reward = 22.00, steps = 22\n",
      "09:26:28 [DEBUG] train episode 467: reward = 200.00, steps = 200\n",
      "09:26:45 [DEBUG] train episode 468: reward = 200.00, steps = 200\n",
      "09:27:03 [DEBUG] train episode 469: reward = 200.00, steps = 200\n",
      "09:27:11 [DEBUG] train episode 470: reward = 86.00, steps = 86\n",
      "09:27:25 [DEBUG] train episode 471: reward = 167.00, steps = 167\n",
      "09:27:31 [DEBUG] train episode 472: reward = 59.00, steps = 59\n",
      "09:27:47 [DEBUG] train episode 473: reward = 176.00, steps = 176\n",
      "09:28:04 [DEBUG] train episode 474: reward = 200.00, steps = 200\n",
      "09:28:22 [DEBUG] train episode 475: reward = 200.00, steps = 200\n",
      "09:28:40 [DEBUG] train episode 476: reward = 200.00, steps = 200\n",
      "09:28:57 [DEBUG] train episode 477: reward = 200.00, steps = 200\n",
      "09:29:11 [DEBUG] train episode 478: reward = 153.00, steps = 153\n",
      "09:29:29 [DEBUG] train episode 479: reward = 200.00, steps = 200\n",
      "09:29:46 [DEBUG] train episode 480: reward = 200.00, steps = 200\n",
      "09:29:58 [DEBUG] train episode 481: reward = 138.00, steps = 138\n",
      "09:30:12 [DEBUG] train episode 482: reward = 155.00, steps = 155\n",
      "09:30:31 [DEBUG] train episode 483: reward = 200.00, steps = 200\n",
      "09:30:43 [DEBUG] train episode 484: reward = 128.00, steps = 128\n",
      "09:31:02 [DEBUG] train episode 485: reward = 200.00, steps = 200\n",
      "09:31:20 [DEBUG] train episode 486: reward = 200.00, steps = 200\n",
      "09:31:38 [DEBUG] train episode 487: reward = 200.00, steps = 200\n",
      "09:31:53 [DEBUG] train episode 488: reward = 153.00, steps = 153\n",
      "09:31:59 [DEBUG] train episode 489: reward = 64.00, steps = 64\n",
      "09:32:17 [DEBUG] train episode 490: reward = 200.00, steps = 200\n",
      "09:32:36 [DEBUG] train episode 491: reward = 200.00, steps = 200\n",
      "09:32:42 [DEBUG] train episode 492: reward = 62.00, steps = 62\n",
      "09:32:50 [DEBUG] train episode 493: reward = 87.00, steps = 87\n",
      "09:33:03 [DEBUG] train episode 494: reward = 144.00, steps = 144\n",
      "09:33:22 [DEBUG] train episode 495: reward = 200.00, steps = 200\n",
      "09:33:37 [DEBUG] train episode 496: reward = 168.00, steps = 168\n",
      "09:33:56 [DEBUG] train episode 497: reward = 200.00, steps = 200\n",
      "09:34:07 [DEBUG] train episode 498: reward = 121.00, steps = 121\n",
      "09:34:18 [DEBUG] train episode 499: reward = 112.00, steps = 112\n",
      "09:34:32 [DEBUG] train episode 500: reward = 151.00, steps = 151\n",
      "09:34:46 [DEBUG] train episode 501: reward = 155.00, steps = 155\n",
      "09:35:04 [DEBUG] train episode 502: reward = 200.00, steps = 200\n",
      "09:35:17 [DEBUG] train episode 503: reward = 141.00, steps = 141\n",
      "09:35:35 [DEBUG] train episode 504: reward = 200.00, steps = 200\n",
      "09:35:49 [DEBUG] train episode 505: reward = 150.00, steps = 150\n",
      "09:36:01 [DEBUG] train episode 506: reward = 139.00, steps = 139\n",
      "09:36:20 [DEBUG] train episode 507: reward = 200.00, steps = 200\n",
      "09:36:39 [DEBUG] train episode 508: reward = 200.00, steps = 200\n",
      "09:36:51 [DEBUG] train episode 509: reward = 126.00, steps = 126\n",
      "09:37:09 [DEBUG] train episode 510: reward = 200.00, steps = 200\n",
      "09:37:27 [DEBUG] train episode 511: reward = 200.00, steps = 200\n",
      "09:37:46 [DEBUG] train episode 512: reward = 200.00, steps = 200\n",
      "09:37:58 [DEBUG] train episode 513: reward = 136.00, steps = 136\n",
      "09:38:04 [DEBUG] train episode 514: reward = 57.00, steps = 57\n",
      "09:38:14 [DEBUG] train episode 515: reward = 109.00, steps = 109\n",
      "09:38:33 [DEBUG] train episode 516: reward = 196.00, steps = 196\n",
      "09:38:49 [DEBUG] train episode 517: reward = 166.00, steps = 166\n",
      "09:39:08 [DEBUG] train episode 518: reward = 200.00, steps = 200\n",
      "09:39:19 [DEBUG] train episode 519: reward = 117.00, steps = 117\n",
      "09:39:35 [DEBUG] train episode 520: reward = 163.00, steps = 163\n",
      "09:39:50 [DEBUG] train episode 521: reward = 164.00, steps = 164\n",
      "09:40:03 [DEBUG] train episode 522: reward = 143.00, steps = 143\n",
      "09:40:12 [DEBUG] train episode 523: reward = 105.00, steps = 105\n",
      "09:40:31 [DEBUG] train episode 524: reward = 196.00, steps = 196\n",
      "09:40:47 [DEBUG] train episode 525: reward = 154.00, steps = 154\n",
      "09:40:57 [DEBUG] train episode 526: reward = 99.00, steps = 99\n",
      "09:41:08 [DEBUG] train episode 527: reward = 124.00, steps = 124\n",
      "09:41:19 [DEBUG] train episode 528: reward = 120.00, steps = 120\n",
      "09:41:31 [DEBUG] train episode 529: reward = 140.00, steps = 140\n",
      "09:41:48 [DEBUG] train episode 530: reward = 180.00, steps = 180\n",
      "09:42:04 [DEBUG] train episode 531: reward = 180.00, steps = 180\n",
      "09:42:14 [DEBUG] train episode 532: reward = 115.00, steps = 115\n",
      "09:42:24 [DEBUG] train episode 533: reward = 106.00, steps = 106\n",
      "09:42:39 [DEBUG] train episode 534: reward = 165.00, steps = 165\n",
      "09:42:54 [DEBUG] train episode 535: reward = 159.00, steps = 159\n",
      "09:42:59 [DEBUG] train episode 536: reward = 56.00, steps = 56\n",
      "09:43:09 [DEBUG] train episode 537: reward = 108.00, steps = 108\n",
      "09:43:24 [DEBUG] train episode 538: reward = 152.00, steps = 152\n",
      "09:43:34 [DEBUG] train episode 539: reward = 112.00, steps = 112\n",
      "09:43:50 [DEBUG] train episode 540: reward = 175.00, steps = 175\n",
      "09:44:05 [DEBUG] train episode 541: reward = 166.00, steps = 166\n",
      "09:44:20 [DEBUG] train episode 542: reward = 149.00, steps = 149\n",
      "09:44:31 [DEBUG] train episode 543: reward = 126.00, steps = 126\n",
      "09:44:45 [DEBUG] train episode 544: reward = 145.00, steps = 145\n",
      "09:45:03 [DEBUG] train episode 545: reward = 200.00, steps = 200\n",
      "09:45:17 [DEBUG] train episode 546: reward = 157.00, steps = 157\n",
      "09:45:35 [DEBUG] train episode 547: reward = 200.00, steps = 200\n",
      "09:45:46 [DEBUG] train episode 548: reward = 125.00, steps = 125\n",
      "09:45:55 [DEBUG] train episode 549: reward = 98.00, steps = 98\n",
      "09:46:07 [DEBUG] train episode 550: reward = 131.00, steps = 131\n",
      "09:46:23 [DEBUG] train episode 551: reward = 189.00, steps = 189\n",
      "09:46:41 [DEBUG] train episode 552: reward = 200.00, steps = 200\n",
      "09:46:58 [DEBUG] train episode 553: reward = 200.00, steps = 200\n",
      "09:47:13 [DEBUG] train episode 554: reward = 161.00, steps = 161\n",
      "09:47:31 [DEBUG] train episode 555: reward = 200.00, steps = 200\n",
      "09:47:50 [DEBUG] train episode 556: reward = 200.00, steps = 200\n",
      "09:48:05 [DEBUG] train episode 557: reward = 163.00, steps = 163\n",
      "09:48:16 [DEBUG] train episode 558: reward = 119.00, steps = 119\n",
      "09:48:34 [DEBUG] train episode 559: reward = 200.00, steps = 200\n",
      "09:48:52 [DEBUG] train episode 560: reward = 200.00, steps = 200\n",
      "09:49:06 [DEBUG] train episode 561: reward = 157.00, steps = 157\n",
      "09:49:24 [DEBUG] train episode 562: reward = 187.00, steps = 187\n",
      "09:49:40 [DEBUG] train episode 563: reward = 183.00, steps = 183\n",
      "09:49:58 [DEBUG] train episode 564: reward = 200.00, steps = 200\n",
      "09:50:13 [DEBUG] train episode 565: reward = 157.00, steps = 157\n",
      "09:50:27 [DEBUG] train episode 566: reward = 158.00, steps = 158\n",
      "09:50:44 [DEBUG] train episode 567: reward = 200.00, steps = 200\n",
      "09:51:01 [DEBUG] train episode 568: reward = 183.00, steps = 183\n",
      "09:51:19 [DEBUG] train episode 569: reward = 199.00, steps = 199\n",
      "09:51:37 [DEBUG] train episode 570: reward = 200.00, steps = 200\n",
      "09:51:55 [DEBUG] train episode 571: reward = 200.00, steps = 200\n",
      "09:52:11 [DEBUG] train episode 572: reward = 180.00, steps = 180\n",
      "09:52:24 [DEBUG] train episode 573: reward = 142.00, steps = 142\n",
      "09:52:42 [DEBUG] train episode 574: reward = 200.00, steps = 200\n",
      "09:53:00 [DEBUG] train episode 575: reward = 200.00, steps = 200\n",
      "09:53:16 [DEBUG] train episode 576: reward = 183.00, steps = 183\n",
      "09:53:34 [DEBUG] train episode 577: reward = 200.00, steps = 200\n",
      "09:53:45 [DEBUG] train episode 578: reward = 126.00, steps = 126\n",
      "09:53:55 [DEBUG] train episode 579: reward = 119.00, steps = 119\n",
      "09:54:07 [DEBUG] train episode 580: reward = 137.00, steps = 137\n",
      "09:54:26 [DEBUG] train episode 581: reward = 200.00, steps = 200\n",
      "09:54:43 [DEBUG] train episode 582: reward = 200.00, steps = 200\n",
      "09:54:57 [DEBUG] train episode 583: reward = 154.00, steps = 154\n",
      "09:55:16 [DEBUG] train episode 584: reward = 200.00, steps = 200\n",
      "09:55:34 [DEBUG] train episode 585: reward = 200.00, steps = 200\n",
      "09:55:52 [DEBUG] train episode 586: reward = 200.00, steps = 200\n",
      "09:56:09 [DEBUG] train episode 587: reward = 185.00, steps = 185\n",
      "09:56:27 [DEBUG] train episode 588: reward = 200.00, steps = 200\n",
      "09:56:46 [DEBUG] train episode 589: reward = 200.00, steps = 200\n",
      "09:56:55 [DEBUG] train episode 590: reward = 92.00, steps = 92\n",
      "09:57:09 [DEBUG] train episode 591: reward = 149.00, steps = 149\n",
      "09:57:19 [DEBUG] train episode 592: reward = 111.00, steps = 111\n",
      "09:57:38 [DEBUG] train episode 593: reward = 200.00, steps = 200\n",
      "09:57:56 [DEBUG] train episode 594: reward = 200.00, steps = 200\n",
      "09:58:09 [DEBUG] train episode 595: reward = 151.00, steps = 151\n",
      "09:58:23 [DEBUG] train episode 596: reward = 159.00, steps = 159\n",
      "09:58:36 [DEBUG] train episode 597: reward = 147.00, steps = 147\n",
      "09:58:54 [DEBUG] train episode 598: reward = 200.00, steps = 200\n",
      "09:59:12 [DEBUG] train episode 599: reward = 200.00, steps = 200\n",
      "09:59:28 [DEBUG] train episode 600: reward = 171.00, steps = 171\n",
      "09:59:31 [DEBUG] train episode 601: reward = 27.00, steps = 27\n",
      "09:59:49 [DEBUG] train episode 602: reward = 200.00, steps = 200\n",
      "09:59:55 [DEBUG] train episode 603: reward = 73.00, steps = 73\n",
      "10:00:10 [DEBUG] train episode 604: reward = 166.00, steps = 166\n",
      "10:00:24 [DEBUG] train episode 605: reward = 152.00, steps = 152\n",
      "10:00:42 [DEBUG] train episode 606: reward = 200.00, steps = 200\n",
      "10:01:01 [DEBUG] train episode 607: reward = 200.00, steps = 200\n",
      "10:01:19 [DEBUG] train episode 608: reward = 200.00, steps = 200\n",
      "10:01:37 [DEBUG] train episode 609: reward = 200.00, steps = 200\n",
      "10:01:48 [DEBUG] train episode 610: reward = 119.00, steps = 119\n",
      "10:02:06 [DEBUG] train episode 611: reward = 200.00, steps = 200\n",
      "10:02:20 [DEBUG] train episode 612: reward = 148.00, steps = 148\n",
      "10:02:37 [DEBUG] train episode 613: reward = 182.00, steps = 182\n",
      "10:02:55 [DEBUG] train episode 614: reward = 200.00, steps = 200\n",
      "10:03:14 [DEBUG] train episode 615: reward = 200.00, steps = 200\n",
      "10:03:33 [DEBUG] train episode 616: reward = 200.00, steps = 200\n",
      "10:03:46 [DEBUG] train episode 617: reward = 149.00, steps = 149\n",
      "10:03:59 [DEBUG] train episode 618: reward = 135.00, steps = 135\n",
      "10:04:11 [DEBUG] train episode 619: reward = 128.00, steps = 128\n",
      "10:04:24 [DEBUG] train episode 620: reward = 131.00, steps = 131\n",
      "10:04:43 [DEBUG] train episode 621: reward = 200.00, steps = 200\n",
      "10:05:01 [DEBUG] train episode 622: reward = 200.00, steps = 200\n",
      "10:05:21 [DEBUG] train episode 623: reward = 200.00, steps = 200\n",
      "10:05:40 [DEBUG] train episode 624: reward = 200.00, steps = 200\n",
      "10:05:54 [DEBUG] train episode 625: reward = 133.00, steps = 133\n",
      "10:06:14 [DEBUG] train episode 626: reward = 200.00, steps = 200\n",
      "10:06:30 [DEBUG] train episode 627: reward = 152.00, steps = 152\n",
      "10:06:49 [DEBUG] train episode 628: reward = 200.00, steps = 200\n",
      "10:07:00 [DEBUG] train episode 629: reward = 112.00, steps = 112\n",
      "10:07:15 [DEBUG] train episode 630: reward = 159.00, steps = 159\n",
      "10:07:28 [DEBUG] train episode 631: reward = 140.00, steps = 140\n",
      "10:07:47 [DEBUG] train episode 632: reward = 200.00, steps = 200\n",
      "10:08:01 [DEBUG] train episode 633: reward = 144.00, steps = 144\n",
      "10:08:14 [DEBUG] train episode 634: reward = 140.00, steps = 140\n",
      "10:08:33 [DEBUG] train episode 635: reward = 200.00, steps = 200\n",
      "10:08:45 [DEBUG] train episode 636: reward = 130.00, steps = 130\n",
      "10:09:05 [DEBUG] train episode 637: reward = 200.00, steps = 200\n",
      "10:09:20 [DEBUG] train episode 638: reward = 162.00, steps = 162\n",
      "10:09:39 [DEBUG] train episode 639: reward = 200.00, steps = 200\n",
      "10:09:57 [DEBUG] train episode 640: reward = 200.00, steps = 200\n",
      "10:10:10 [DEBUG] train episode 641: reward = 161.00, steps = 161\n",
      "10:10:20 [DEBUG] train episode 642: reward = 117.00, steps = 117\n",
      "10:10:27 [DEBUG] train episode 643: reward = 75.00, steps = 75\n",
      "10:10:43 [DEBUG] train episode 644: reward = 200.00, steps = 200\n",
      "10:11:00 [DEBUG] train episode 645: reward = 200.00, steps = 200\n",
      "10:11:20 [DEBUG] train episode 646: reward = 200.00, steps = 200\n",
      "10:11:39 [DEBUG] train episode 647: reward = 200.00, steps = 200\n",
      "10:11:59 [DEBUG] train episode 648: reward = 200.00, steps = 200\n",
      "10:12:16 [DEBUG] train episode 649: reward = 170.00, steps = 170\n",
      "10:12:31 [DEBUG] train episode 650: reward = 140.00, steps = 140\n",
      "10:12:50 [DEBUG] train episode 651: reward = 200.00, steps = 200\n",
      "10:13:00 [DEBUG] train episode 652: reward = 99.00, steps = 99\n",
      "10:13:17 [DEBUG] train episode 653: reward = 159.00, steps = 159\n",
      "10:13:35 [DEBUG] train episode 654: reward = 195.00, steps = 195\n",
      "10:13:53 [DEBUG] train episode 655: reward = 200.00, steps = 200\n",
      "10:14:06 [DEBUG] train episode 656: reward = 120.00, steps = 120\n",
      "10:14:24 [DEBUG] train episode 657: reward = 200.00, steps = 200\n",
      "10:14:42 [DEBUG] train episode 658: reward = 200.00, steps = 200\n",
      "10:14:56 [DEBUG] train episode 659: reward = 147.00, steps = 147\n",
      "10:15:09 [DEBUG] train episode 660: reward = 145.00, steps = 145\n",
      "10:15:28 [DEBUG] train episode 661: reward = 200.00, steps = 200\n",
      "10:15:46 [DEBUG] train episode 662: reward = 200.00, steps = 200\n",
      "10:16:05 [DEBUG] train episode 663: reward = 200.00, steps = 200\n",
      "10:16:23 [DEBUG] train episode 664: reward = 200.00, steps = 200\n",
      "10:16:39 [DEBUG] train episode 665: reward = 176.00, steps = 176\n",
      "10:16:52 [DEBUG] train episode 666: reward = 148.00, steps = 148\n",
      "10:17:09 [DEBUG] train episode 667: reward = 190.00, steps = 190\n",
      "10:17:18 [DEBUG] train episode 668: reward = 106.00, steps = 106\n",
      "10:17:37 [DEBUG] train episode 669: reward = 200.00, steps = 200\n",
      "10:17:54 [DEBUG] train episode 670: reward = 200.00, steps = 200\n",
      "10:18:11 [DEBUG] train episode 671: reward = 200.00, steps = 200\n",
      "10:18:27 [DEBUG] train episode 672: reward = 183.00, steps = 183\n",
      "10:18:43 [DEBUG] train episode 673: reward = 200.00, steps = 200\n",
      "10:18:57 [DEBUG] train episode 674: reward = 162.00, steps = 162\n",
      "10:19:13 [DEBUG] train episode 675: reward = 200.00, steps = 200\n",
      "10:19:29 [DEBUG] train episode 676: reward = 188.00, steps = 188\n",
      "10:19:43 [DEBUG] train episode 677: reward = 158.00, steps = 158\n",
      "10:19:59 [DEBUG] train episode 678: reward = 200.00, steps = 200\n",
      "10:20:15 [DEBUG] train episode 679: reward = 162.00, steps = 162\n",
      "10:20:32 [DEBUG] train episode 680: reward = 200.00, steps = 200\n",
      "10:20:47 [DEBUG] train episode 681: reward = 149.00, steps = 149\n",
      "10:21:01 [DEBUG] train episode 682: reward = 140.00, steps = 140\n",
      "10:21:19 [DEBUG] train episode 683: reward = 200.00, steps = 200\n",
      "10:21:32 [DEBUG] train episode 684: reward = 139.00, steps = 139\n",
      "10:21:51 [DEBUG] train episode 685: reward = 200.00, steps = 200\n",
      "10:22:07 [DEBUG] train episode 686: reward = 187.00, steps = 187\n",
      "10:22:25 [DEBUG] train episode 687: reward = 200.00, steps = 200\n",
      "10:22:40 [DEBUG] train episode 688: reward = 172.00, steps = 172\n",
      "10:22:57 [DEBUG] train episode 689: reward = 200.00, steps = 200\n",
      "10:23:15 [DEBUG] train episode 690: reward = 200.00, steps = 200\n",
      "10:23:28 [DEBUG] train episode 691: reward = 142.00, steps = 142\n",
      "10:23:43 [DEBUG] train episode 692: reward = 180.00, steps = 180\n",
      "10:23:58 [DEBUG] train episode 693: reward = 162.00, steps = 162\n",
      "10:24:15 [DEBUG] train episode 694: reward = 200.00, steps = 200\n",
      "10:24:33 [DEBUG] train episode 695: reward = 200.00, steps = 200\n",
      "10:24:47 [DEBUG] train episode 696: reward = 157.00, steps = 157\n",
      "10:25:05 [DEBUG] train episode 697: reward = 200.00, steps = 200\n",
      "10:25:22 [DEBUG] train episode 698: reward = 196.00, steps = 196\n",
      "10:25:39 [DEBUG] train episode 699: reward = 200.00, steps = 200\n",
      "10:25:51 [DEBUG] train episode 700: reward = 139.00, steps = 139\n",
      "10:26:08 [DEBUG] train episode 701: reward = 185.00, steps = 185\n",
      "10:26:28 [DEBUG] train episode 702: reward = 200.00, steps = 200\n",
      "10:26:48 [DEBUG] train episode 703: reward = 200.00, steps = 200\n",
      "10:27:00 [DEBUG] train episode 704: reward = 136.00, steps = 136\n",
      "10:27:14 [DEBUG] train episode 705: reward = 161.00, steps = 161\n",
      "10:27:33 [DEBUG] train episode 706: reward = 200.00, steps = 200\n",
      "10:27:51 [DEBUG] train episode 707: reward = 200.00, steps = 200\n",
      "10:28:08 [DEBUG] train episode 708: reward = 173.00, steps = 173\n",
      "10:28:26 [DEBUG] train episode 709: reward = 200.00, steps = 200\n",
      "10:28:46 [DEBUG] train episode 710: reward = 200.00, steps = 200\n",
      "10:29:04 [DEBUG] train episode 711: reward = 200.00, steps = 200\n",
      "10:29:19 [DEBUG] train episode 712: reward = 162.00, steps = 162\n",
      "10:29:34 [DEBUG] train episode 713: reward = 148.00, steps = 148\n",
      "10:29:53 [DEBUG] train episode 714: reward = 200.00, steps = 200\n",
      "10:30:11 [DEBUG] train episode 715: reward = 200.00, steps = 200\n",
      "10:30:29 [DEBUG] train episode 716: reward = 200.00, steps = 200\n",
      "10:30:48 [DEBUG] train episode 717: reward = 200.00, steps = 200\n",
      "10:30:59 [DEBUG] train episode 718: reward = 123.00, steps = 123\n",
      "10:31:15 [DEBUG] train episode 719: reward = 169.00, steps = 169\n",
      "10:31:33 [DEBUG] train episode 720: reward = 200.00, steps = 200\n",
      "10:31:51 [DEBUG] train episode 721: reward = 200.00, steps = 200\n",
      "10:32:09 [DEBUG] train episode 722: reward = 194.00, steps = 194\n",
      "10:32:28 [DEBUG] train episode 723: reward = 200.00, steps = 200\n",
      "10:32:39 [DEBUG] train episode 724: reward = 129.00, steps = 129\n",
      "10:32:54 [DEBUG] train episode 725: reward = 155.00, steps = 155\n",
      "10:33:13 [DEBUG] train episode 726: reward = 200.00, steps = 200\n",
      "10:33:31 [DEBUG] train episode 727: reward = 200.00, steps = 200\n",
      "10:33:44 [DEBUG] train episode 728: reward = 146.00, steps = 146\n",
      "10:34:00 [DEBUG] train episode 729: reward = 170.00, steps = 170\n",
      "10:34:15 [DEBUG] train episode 730: reward = 165.00, steps = 165\n",
      "10:34:34 [DEBUG] train episode 731: reward = 200.00, steps = 200\n",
      "10:34:50 [DEBUG] train episode 732: reward = 200.00, steps = 200\n",
      "10:35:07 [DEBUG] train episode 733: reward = 200.00, steps = 200\n",
      "10:35:25 [DEBUG] train episode 734: reward = 194.00, steps = 194\n",
      "10:35:42 [DEBUG] train episode 735: reward = 200.00, steps = 200\n",
      "10:35:56 [DEBUG] train episode 736: reward = 163.00, steps = 163\n",
      "10:36:12 [DEBUG] train episode 737: reward = 170.00, steps = 170\n",
      "10:36:29 [DEBUG] train episode 738: reward = 200.00, steps = 200\n",
      "10:36:48 [DEBUG] train episode 739: reward = 198.00, steps = 198\n",
      "10:37:05 [DEBUG] train episode 740: reward = 200.00, steps = 200\n",
      "10:37:24 [DEBUG] train episode 741: reward = 200.00, steps = 200\n",
      "10:37:44 [DEBUG] train episode 742: reward = 200.00, steps = 200\n",
      "10:38:02 [DEBUG] train episode 743: reward = 200.00, steps = 200\n",
      "10:38:13 [DEBUG] train episode 744: reward = 122.00, steps = 122\n",
      "10:38:26 [DEBUG] train episode 745: reward = 145.00, steps = 145\n",
      "10:38:44 [DEBUG] train episode 746: reward = 200.00, steps = 200\n",
      "10:39:06 [DEBUG] train episode 747: reward = 200.00, steps = 200\n",
      "10:39:27 [DEBUG] train episode 748: reward = 200.00, steps = 200\n",
      "10:39:48 [DEBUG] train episode 749: reward = 200.00, steps = 200\n",
      "10:40:09 [DEBUG] train episode 750: reward = 200.00, steps = 200\n",
      "10:40:29 [DEBUG] train episode 751: reward = 200.00, steps = 200\n",
      "10:40:50 [DEBUG] train episode 752: reward = 200.00, steps = 200\n",
      "10:41:13 [DEBUG] train episode 753: reward = 200.00, steps = 200\n",
      "10:41:32 [DEBUG] train episode 754: reward = 200.00, steps = 200\n",
      "10:41:51 [DEBUG] train episode 755: reward = 200.00, steps = 200\n",
      "10:41:52 [INFO] ==== test ====\n",
      "10:42:47 [DEBUG] test episode 0: reward = 200.00, steps = 200\n",
      "10:44:05 [DEBUG] test episode 1: reward = 200.00, steps = 200\n",
      "10:45:23 [DEBUG] test episode 2: reward = 200.00, steps = 200\n",
      "10:46:40 [DEBUG] test episode 3: reward = 200.00, steps = 200\n",
      "10:47:57 [DEBUG] test episode 4: reward = 200.00, steps = 200\n",
      "10:49:13 [DEBUG] test episode 5: reward = 200.00, steps = 200\n",
      "10:50:30 [DEBUG] test episode 6: reward = 200.00, steps = 200\n",
      "10:51:49 [DEBUG] test episode 7: reward = 200.00, steps = 200\n",
      "10:53:07 [DEBUG] test episode 8: reward = 200.00, steps = 200\n",
      "10:54:27 [DEBUG] test episode 9: reward = 200.00, steps = 200\n",
      "10:55:46 [DEBUG] test episode 10: reward = 200.00, steps = 200\n",
      "10:57:02 [DEBUG] test episode 11: reward = 200.00, steps = 200\n",
      "10:58:19 [DEBUG] test episode 12: reward = 200.00, steps = 200\n",
      "10:59:27 [DEBUG] test episode 13: reward = 179.00, steps = 179\n",
      "11:00:13 [DEBUG] test episode 14: reward = 119.00, steps = 119\n",
      "11:01:30 [DEBUG] test episode 15: reward = 200.00, steps = 200\n",
      "11:02:47 [DEBUG] test episode 16: reward = 200.00, steps = 200\n",
      "11:04:09 [DEBUG] test episode 17: reward = 200.00, steps = 200\n",
      "11:05:02 [DEBUG] test episode 18: reward = 140.00, steps = 140\n",
      "11:06:21 [DEBUG] test episode 19: reward = 200.00, steps = 200\n",
      "11:06:53 [DEBUG] test episode 20: reward = 86.00, steps = 86\n",
      "11:08:10 [DEBUG] test episode 21: reward = 200.00, steps = 200\n",
      "11:09:27 [DEBUG] test episode 22: reward = 200.00, steps = 200\n",
      "11:10:48 [DEBUG] test episode 23: reward = 200.00, steps = 200\n",
      "11:12:07 [DEBUG] test episode 24: reward = 200.00, steps = 200\n",
      "11:13:27 [DEBUG] test episode 25: reward = 200.00, steps = 200\n",
      "11:14:44 [DEBUG] test episode 26: reward = 200.00, steps = 200\n",
      "11:16:00 [DEBUG] test episode 27: reward = 200.00, steps = 200\n",
      "11:17:19 [DEBUG] test episode 28: reward = 200.00, steps = 200\n",
      "11:18:35 [DEBUG] test episode 29: reward = 200.00, steps = 200\n",
      "11:19:51 [DEBUG] test episode 30: reward = 200.00, steps = 200\n",
      "11:21:12 [DEBUG] test episode 31: reward = 200.00, steps = 200\n",
      "11:22:27 [DEBUG] test episode 32: reward = 200.00, steps = 200\n",
      "11:23:42 [DEBUG] test episode 33: reward = 200.00, steps = 200\n",
      "11:24:58 [DEBUG] test episode 34: reward = 200.00, steps = 200\n",
      "11:26:14 [DEBUG] test episode 35: reward = 200.00, steps = 200\n",
      "11:27:29 [DEBUG] test episode 36: reward = 199.00, steps = 199\n",
      "11:28:39 [DEBUG] test episode 37: reward = 182.00, steps = 182\n",
      "11:29:54 [DEBUG] test episode 38: reward = 200.00, steps = 200\n",
      "11:31:11 [DEBUG] test episode 39: reward = 200.00, steps = 200\n",
      "11:32:26 [DEBUG] test episode 40: reward = 200.00, steps = 200\n",
      "11:33:44 [DEBUG] test episode 41: reward = 200.00, steps = 200\n",
      "11:34:59 [DEBUG] test episode 42: reward = 200.00, steps = 200\n",
      "11:36:15 [DEBUG] test episode 43: reward = 200.00, steps = 200\n",
      "11:37:05 [DEBUG] test episode 44: reward = 131.00, steps = 131\n",
      "11:38:22 [DEBUG] test episode 45: reward = 200.00, steps = 200\n",
      "11:39:08 [DEBUG] test episode 46: reward = 121.00, steps = 121\n",
      "11:40:24 [DEBUG] test episode 47: reward = 200.00, steps = 200\n",
      "11:41:39 [DEBUG] test episode 48: reward = 200.00, steps = 200\n",
      "11:42:56 [DEBUG] test episode 49: reward = 200.00, steps = 200\n",
      "11:44:12 [DEBUG] test episode 50: reward = 200.00, steps = 200\n",
      "11:45:25 [DEBUG] test episode 51: reward = 200.00, steps = 200\n",
      "11:46:42 [DEBUG] test episode 52: reward = 200.00, steps = 200\n",
      "11:47:36 [DEBUG] test episode 53: reward = 141.00, steps = 141\n",
      "11:48:53 [DEBUG] test episode 54: reward = 200.00, steps = 200\n",
      "11:50:08 [DEBUG] test episode 55: reward = 200.00, steps = 200\n",
      "11:51:24 [DEBUG] test episode 56: reward = 200.00, steps = 200\n",
      "11:52:39 [DEBUG] test episode 57: reward = 200.00, steps = 200\n",
      "11:53:55 [DEBUG] test episode 58: reward = 200.00, steps = 200\n",
      "11:55:11 [DEBUG] test episode 59: reward = 200.00, steps = 200\n",
      "11:56:26 [DEBUG] test episode 60: reward = 200.00, steps = 200\n",
      "11:57:41 [DEBUG] test episode 61: reward = 200.00, steps = 200\n",
      "11:58:58 [DEBUG] test episode 62: reward = 200.00, steps = 200\n",
      "12:00:14 [DEBUG] test episode 63: reward = 200.00, steps = 200\n",
      "12:01:29 [DEBUG] test episode 64: reward = 200.00, steps = 200\n",
      "12:02:31 [DEBUG] test episode 65: reward = 159.00, steps = 159\n",
      "12:03:47 [DEBUG] test episode 66: reward = 200.00, steps = 200\n",
      "12:05:02 [DEBUG] test episode 67: reward = 200.00, steps = 200\n",
      "12:06:18 [DEBUG] test episode 68: reward = 200.00, steps = 200\n",
      "12:07:35 [DEBUG] test episode 69: reward = 200.00, steps = 200\n",
      "12:08:53 [DEBUG] test episode 70: reward = 200.00, steps = 200\n",
      "12:10:08 [DEBUG] test episode 71: reward = 200.00, steps = 200\n",
      "12:11:22 [DEBUG] test episode 72: reward = 200.00, steps = 200\n",
      "12:12:38 [DEBUG] test episode 73: reward = 200.00, steps = 200\n",
      "12:13:55 [DEBUG] test episode 74: reward = 200.00, steps = 200\n",
      "12:15:09 [DEBUG] test episode 75: reward = 200.00, steps = 200\n",
      "12:16:25 [DEBUG] test episode 76: reward = 200.00, steps = 200\n",
      "12:17:42 [DEBUG] test episode 77: reward = 200.00, steps = 200\n",
      "12:18:59 [DEBUG] test episode 78: reward = 200.00, steps = 200\n",
      "12:20:15 [DEBUG] test episode 79: reward = 200.00, steps = 200\n",
      "12:21:31 [DEBUG] test episode 80: reward = 200.00, steps = 200\n",
      "12:22:47 [DEBUG] test episode 81: reward = 200.00, steps = 200\n",
      "12:24:02 [DEBUG] test episode 82: reward = 200.00, steps = 200\n",
      "12:25:18 [DEBUG] test episode 83: reward = 200.00, steps = 200\n",
      "12:26:34 [DEBUG] test episode 84: reward = 200.00, steps = 200\n",
      "12:27:55 [DEBUG] test episode 85: reward = 200.00, steps = 200\n",
      "12:29:12 [DEBUG] test episode 86: reward = 200.00, steps = 200\n",
      "12:30:29 [DEBUG] test episode 87: reward = 200.00, steps = 200\n",
      "12:31:45 [DEBUG] test episode 88: reward = 200.00, steps = 200\n",
      "12:32:48 [DEBUG] test episode 89: reward = 170.00, steps = 170\n",
      "12:34:04 [DEBUG] test episode 90: reward = 200.00, steps = 200\n",
      "12:35:20 [DEBUG] test episode 91: reward = 200.00, steps = 200\n",
      "12:36:37 [DEBUG] test episode 92: reward = 200.00, steps = 200\n",
      "12:37:31 [DEBUG] test episode 93: reward = 141.00, steps = 141\n",
      "12:38:47 [DEBUG] test episode 94: reward = 200.00, steps = 200\n",
      "12:40:02 [DEBUG] test episode 95: reward = 200.00, steps = 200\n",
      "12:41:18 [DEBUG] test episode 96: reward = 200.00, steps = 200\n",
      "12:42:20 [DEBUG] test episode 97: reward = 162.00, steps = 162\n",
      "12:43:37 [DEBUG] test episode 98: reward = 200.00, steps = 200\n",
      "12:44:54 [DEBUG] test episode 99: reward = 200.00, steps = 200\n",
      "12:44:54 [INFO] average episode reward = 193.30 ± 20.35\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29d5wcR5k3/n16ZnZXGxR3lbMVbFmWJVnOOUcw+DDIvHAmnQ0272HMCzYHvOQ7H/EIB5w50r1kMAb/DGfwGSeCg5yEZFuOsq1g5Sztanenfn9Md091dVV1VXfP7Oyovp+PtDPd1VVP93R/6+lvPfUUMcbg4ODg4NBc8IbaAAcHBweH/OHI3cHBwaEJ4cjdwcHBoQnhyN3BwcGhCeHI3cHBwaEJURxqAwCgu7ubzZw5c6jNcHBwcBhWeOSRR7Yyxnpk+xqC3GfOnIkVK1YMtRkODg4OwwpE9JJqn5NlHBwcHJoQjtwdHBwcmhCO3B0cHByaEI7cHRwcHJoQjtwdHBwcmhCJ5E5E04jobiJ6iohWE9H7/O1jiehOInrW/zuGO+bDRPQcEa0hovNreQIODg4ODnGYeO4DAD7AGDsCwAkAriWiBQBuBHAXY2wugLv87/D3LQdwJIALAHyDiAq1MN7BwcHBQY7EOHfG2EYAG/3Pe4joKQBTAFwK4Ay/2A8A3APgBn/7TxljfQBeJKLnABwH4K95G+8wNHj61d3Y2zuAZTPHAgCe3LAb9z+7BW87eSZai8n9+Cvb9+P5LXtxxvzxuPWxdTj/yIlobyni1V29eOzlHXhh6z7MHd+JzrYiJo5sQ2dbEQ+8sB2vPXoyAOC+Z7bgsZd34k3HTsOEka34xSPr8LrFU9BSrPgq+/oGcOeTm3DekRNwx6pX8folU0BEAADGGH74wEvYvKcPR04ehZ6uFhwzo3Iez2zagx37DuL42eNw7zNb8JfntqJU8OAR0DtQRkvBQ6ngYfLoNjAGbNrdi70HB1AgwqWLp2DV+l24bGmlrX19A/ivv76EcxdMwJzxnegfLOPWR9fjDcdMxYMvbse4zhY8/vJOtJY8nHjYONz/zFa8tG0f9vQNYHxXG6aMGYGnNu5G0SMsmzkWp8+rzFP53d824oUte/HWE2biN0+sx9a9B7Gntx+Lp43G81v2AQBmd3dg855enDKnB3c9tQkXHjUR2/f146EXt2Ha2HY8/spOdLWVcM4R4zGruwN/fHozLl08JfF327S7F79Y8QpOPKwbrUUPtz62HiPbSpgyZgTKjGHG2Hbs2H8QBwcZXtm+HxNHtmHNpj04bW4Ptu3rw8p1u9DRWsSe3n5csmgSVq7bhYVTRmH1+l0Y09GCrrYi+gcZVq/fhTID5k/swrOb9+KSRZOwfscBPPbyDgwyhhGlAi48ahKe2rgbr+7qxc79/ehoLeLUud34w5ObQADmTejCjHHtWLF2O7bvO4hZPR14ccs+zJnQhaXTR+MXK9ZhsMzQUvQwb0IXntu8B3PGd4X39qgRJZx1xHjc/fRm7Nzfj/kTu7Bzfz+6u1rw9Kt74BFhVncHNu/uBRGhu7MFL2zdh6OmjMJL2/bjjPk9+MPqTRgol7Fw8ig8/spOtBY9zBjXgXU7DmCwXA6v67yJXbhk0WTDp88cVpOYiGgmgCUAHgQwwSd+MMY2EtF4v9gUAA9wh63zt4l1XQXgKgCYPn26rd0OQ4gL/u1+AMDamy4GANxwy0r8bf0uLJ42GsfPHpd4/Llfvhe9/WXc8p4T8f6fPYE3HLMNX7j8aPzdN/+C9TsPxMpftnQKfvXoekwbMwJLpo/BJ25bjRe27kOpSJg+th0f+uVKrNu+H9efNx8A8PHbVuOXj6zDvAmdeGbTXkwZPSK0a/OePnzsN6sj9Qfncd6X7wMAfO/tx+Lt33vY6pp8457nAQCTRrXhpDnd+Mvz2/CvdzyNh9dux3ffdiz+668v4dO3P4mDg2V89NerIse+fskU3PrYem39L/7LRSAiXPOjRwEAnkf43B1rEqx6GgDwxTufke59auNulAqE3/3tVcyb0IUjJo3U1nb5t/6Kl7fvB/AMZnV34MWt+xLar+Dm+16Ibfven9dKy/Z0tWLLnr7Itld3HcBDL27H2m37w21f+EP8nH780Ai8sj1+//BoLXq4+vTD8NW7nk20+7t/fhE79vcnlpPhxw++LL2Xefj+Bi5ZNHloyZ2IOgHcAuA6xtjuwBOSFZVsi60Iwhi7GcDNALBs2TK3YsgwxqbdvQCAwbLZz9jbX/Fa9vYNRo5XPQztLZW3gUde2oEl08eg7C8w0z/AsKd3AECFtAME5PDMpr0AgH0HB8J9/YNVj0kFkdinjB6R+KAG2NNXaWvAb2fn/oMAEHpqL2yJE+K6HRXSmjpmBNbtkLezZW8fxne1hd8PHBw0skeHwTLDZv/a9/Yn11ch9grWbjMjdlsMSH6fgTJD/2DyvdXbX0ZPVyt2H+hH34D8d+4fLEe8Zh0OGFwTtS36Y5/97IUoFWobz2JUOxGVUCH2HzHGfuVv3kREk/z9kwBs9revAzCNO3wqgA35mOvQiAj6+Vr10EVPfpsOMoaCV2l8gOtYSoWof1EuAw+v3V6xMYWRaj9GDbGf6+lqBVAhaREPr90BAPA0DYkeqQnZJYExFv5mGmdNcWzm5qWQ+QembQXORXBPqOo39EG0v0cSZE7E7J6OXOo2hUm0DAH4DoCnGGNf4nbdBuBK//OVAH7DbV9ORK1ENAvAXAAP5WeyQ6PC5CEMvHQbBA9t4LGzcHsZRf9BHoyQe/S2/spdz+Lyb/0Vj7y0I6zDBqoHccroEbFtQfVBOwFpjmwrAQC27FGfv46UNu/uxeoNu8LvMg/XFgzAynWVOj0CXtiyF3v7BvQH1Riy36fMGEyWAx0sMxCAQgJx5nHtTGzhsWzGGBw+sSv8rvmpc4OJ534ygLcCOIuIHvf/XQTgJgDnEtGzAM71v4MxthrAzwE8CeAOANcyxrK/Qzo0LMhX4piB7378P99lXf+A/xodPN/B34GyynOP3tZPbtwNANi+76Cx18ZD9SC2FmWPT7QjEg/dvu+gsh0dJz2xbhcu/uqfwu8DaU5EwF+e3xZ+9ohw1hfvxf/69gOaI2oPGYebetvlMgMRUCjomdP0rSeNIxC2IRhc8CjydmT7ppQGJtEyf4JcRweAsxXHfBbAZzPY5TCMENynOfCNFAODUY89eOgGB1ko2fA6alF4uAMvqlgg5QNb1hjvKdhdtl3sgELJyv+u03F1HucGQfMfMNSNdTjI6dJB00+s26UoXR8oPXcDx2GQMRAo2XM3vHamY0jSNoS3g2KBQhKtA68DcDNUHXKEyaszD9N7PCbLyDx3zhsrKTT6kucpbdx7UC1HqGQZHYlUPfdomWAw2aYdID6eMZCD5s5DtFNapg6kJCN3xpiR4xDcJ6rOOIDptcvydiQe6lHVc6+H3g44cnfIAcGtWqsB1eAhE/XsQY7ceVIQPXd+u+p5ZRpnTsUVsme0+nYh396riXLRkZLYKR3MWTdW9Id1h+z3KZfNxnPKrCLLFBPI3VSWyTpo3NPVGtpS8KrdZz30dsCRu0OeqBG7i6/HAZEPlFn48Og09+p2tSyje+1Xeu7aqIxAl/Hr97/3DmhkGc3TWGvP3cSbrAcnyd6sGOwGVJPOJQ9JywQFIhzW0xl+Dm6XeujtgCN3hxwQzv60ZPegdNJzG4SVBQ94wOOD5XJIsNFoGYXn7nlQPdc6G2zIvaq5RwdUg+p1XqNWKxYOy5ugTLzJepCSTOcuMzO/ocwqNuo6XSCfwWgTFDwK38b4AVXnuTsMO9i+xuoGMXlUNfegHYnnzpFmUeEC6wZUdVBJFjKyY4jaasOHWllGoLc84twbEfI4dzPPPUASuf925UZbs1LB86pEzssyJuMbubRfl1YcDgnY8qapByVq7iz03JnCc5ff1rqHSmeJekBVUo8Y5x6EiRqcqk5O+N3fXo18zztWu1aTkvKAzcQjIJnc64UC9xbheRS+xtXLvIZYINtheOCDv3gCr2omIdnyg+k08IC4mRBDPlBmoffMyxQlEy1cgM4zVMkROjKOk1Hy1UkK4eORt7RgUt1QUabpJCag8qZkcx1rCc+j8B4pcp/rFS3jyN3BGL94ZJ12v20opClBVTX3yvfgsP6BMt74H5Vko7xKoZJlGMxC6kSo+gppnHvwN5yhGnxPbsfmmTfJkWMDE7lqqDizzMzfLIiSQyHrBY8bRC2Qi3N3GIZIm1tG9MhVqMoyUc+dnxDEvwWoBlT5Y0XoLFB5gto493KU3E1gIyfkHS2TZTZmrcHnwEkCgZShkLr7ohYQZZngXqhX5+PI3SE32PKD6QzAQF8Wo2t479WE7BhTv12kiZaRDbSKET3V1AzJsCF3cXp7VjQwt6PMmFXnoyJPVQK6WkGUZYJ7oV5djCN3h8yocl9tZJmDQfoBySSmAPxnHQ+k4USV963X3O1lGRstNu8BVZOONo8oDz4zoilMJzEBgeYu36ea3FYrFLzqb+p5FDoDboaqw7BDrT33MmO46b+fxn5/licfDhghd00nowq/1B2j8qhlD6kY0WNSf1I7MgxXWWbx1NE46/DxyQU5mOaWASpeseo6ttQ4f7ooB/GyTEXC8z13R+4OwwU20gOPwHNP8ggD+WXr3j58697nueM5WYYjbWWKAV1InaUs85qjJ+MjFx+hPCZdamHzsv05T2Iy6mct7Dth9tg8qgGQ8LuJdWsmMdV6cQyxXSJxElNluwuFdBh2sPbcQy3dbEBVfMAHVJ67pjrbiB5ALst8dfli7O6NJxuLT2LKJ85dRCN77uccMQEjWtRr6do6rmXGjD0Hnedea1mmVPAiK0AVPIpOYvLH/50s4zDsYJt+wJSfVFIHnzzLRJbRhULqTJG9RhMX5iaDmM/d5FSHUnPn5aqnX92Nz9z+JLYKq0aZWlfw9HMNbH136wFVxXWst+deoGr64QI3uFqvUEjnuTukxuoNu7Bzf7/VoCEPU4KqpvqNNhCJljEeULWPllE5fNL0A2FHVNtQyLyjZQa5C/D1Pz6H21duxMIpo/C6JbG17RPhGeR3sYFpbhkAAKmzQtY6FFKs3/OgkGXcJCaHBkewMtDMce0A7F/tTaNlqvHwUfQP8LJMleh14Y55LrMn2ypOtEqyKdKO1YBq7dIPBAPW4rUy5SQtuZO958osPHedLFNrz10MtYzIMo04iYmIvktEm4loFbftZ9ySe2uJ6HF/+0wiOsDt+1YtjXeoHTbv6cUDL1SXYdORSdrR/wEhxFEFMWFYAN5z37G/H9t8GUFVH9PuUxthk34gqEW1zJ4ONo5lLTV3xuS/i2kopCcsKSfCekBVYouybqKGkWX4To6/Jo2kuX8fwAX8BsbYmxhjixljiwHcAuBX3O7ng32MsXfnZ6pDPXHp1/+M5TdX19M0WRzCPhQymlZAXa98QFW06Qp//U9ddalkGWVWSF075vUHsPHc+7l0x3mAt1fsoGzhkSY3PUv2XMX9Nsvd6T332pKqOGDLk3iBGjBahjF2HxHNlO2jSlf0RgBn5WuWw1Bj465ogrA+zfJw1UHDdLJMEokEerBYSvRen9m0t1JOGQqZNreMQpbRPKRMYXOadmQYLDMUiDCY0wop0uXt/L+7e/uxfseB2H4VChQdPLR9A/CIImMAtmuZKqNlajxDNRbnzn2vrKFa3zj3rJr7qQA2Mcae5bbNIqLHAOwG8FHG2P2yA4noKgBXAcD06dMzmuFQa5gkqkqb8jfp4Q0iOXSyDMAv1K2ZxJQit4xac5cNqEY7rHCANeeskGWmX7nJFmXJgHRwLm//3sN45KUdGFFShzfy4KWRAhEGLLV7jwB+vSqbNwgijederK/mzt83Hue5N4zmnoArAPyE+74RwHTG2BIA1wP4MRGNlB3IGLuZMbaMMbasp6cnoxkOtYaWf1NHy5h5t1WCjGJQJA3Iy4Hbrh5sVVuhkktkm8uChCSSvA42jmWZsVxzlMhkmcDmR17aAcA8E6XHEazs2iXLMsR9rjoBRqtFgZSdZEudZZmCV72GBW7hjkbS3KUgoiKAywD8LNjGGOtjjG3zPz8C4HkA87Ia6TD0MPE8bQWCquae4LkrQiGVh2mjZexsBHQLZMd3iKSeYJLQjvlDzwy0axvIBlSDc2krVWjCNLpJjBIRkSzLVD+XPI/LsJl8wjrPveayTCEeLVMld29YLbN3DoCnGWNhkm8i6iGigv95NoC5AF7IZqJDI8CEnGxnf4ZRMAnlBg293+parro27QdUbUIhY3KMxSWxHSDNc7k21cLUAIzlmABE1ciQ2DkREsNl+OtdLJCV5y5t00dessyFCyfi629eEtsuau58Z1Sg6mk3jOdORD8B8FcA84loHRG909+1HFFJBgBOA7CSiJ4A8EsA72aMbc/TYIehgU73TJJDVDAl7bIhUQbPltrWdAOqqmdRtj303IVc9SbN1uuhl4FXXMRY/TZLcvciRBbfn3SWkSgTj6qeu0Uopgz6WbNVfO2KOHHzGN1eQldbKbZdljgs+P0L/EWpE0yiZa5QbH+bZNstqIRGOjQZjDxQS+KsEmBS22ZRNSbrlabJLaOOltENqAbfzdu1Jfc8+wJ+/CKU4Fg6z50nspgXzZLlFX53qeCFtpmcL5FusQ4zz33CyLbENmRyk6i5P79lLyaPHgGgIsvUe5k9l1vGITOqcoitLGPmkqtmqMYNgbYcY7owSXW1NmpJWQjvDGUag2Nto1/ypAi+8xE991Zrz70a9ldIoXNHZBmPwt/fiNyhJs9S0eyKJbVTIJIOfovnunrDbvT6q4V1tBaqbzN1Yl1H7g5GMHF4n9q4BzNv/C0efXmHUZ2BFJBUta0skyq3TIp87vL6AxvMo2QC2C6/lme8dFlC7sE5BAOq5nZBG/aXZDV/vUsFLyR3E483jwHVpJ/BI7ktsjeGA34qh662Yt1zyzhyd9CiOhknWXO/Z81mAMDvV71qVLdNmKBvjXZvVZZREXjalZhsyF2QZfjGE2D70O/ti6cclqFdk343AJ8eXkxb3Fa0lGVIr46bxLkH4AdUTa+OcrEOwwHVpN/b8+S5c2Tb9vnk3tlaqvskJkfuDlrYRH3wRcplhmWfuRO/fGSdsnzZoOOIltfvD7NTquxjaaNlkm0LIMoxoXyd8yQmG5hMApJ67v53W8+dfwOJnZHBuCIJskx+oZCGskzCfk+R7llW//6DlQ64s7UYHlOvcVWXFdJBCyb81ZblCu3YfxBb9x7EZ377JN5wzFRp+VBLN/SmeY+8wGmxAQLPVzd4qfPqVbDxqKtx7kG95udoK8sY22Qw9yhC7v7fT9/+JEoFsk64lSjLJHnGqgFVk7ahzrNvSu5Jvzefmz1ynJTcq7JMcDGGQ5y7wyGAqnaskWVCj7laZoufobGns1V5nK0sw3O5fHIMtPXlnVtGBnHClU17tfLcxZm8Mtxwy9+qX7ji//c3q63JvcANqMpgHwrpH2cULWMX3aSqI2m/7O1A7Dx+/A/HY39f1XMP9jpZxqEhYOK5y0IQN+/2yb1LTe7GUTA+eO9SNzamI1RpgizGpJ3XDD9Pvc2zWI00iZK8yTnWyqOzTbwlwnpyFSVcs0TNnZNlOM/d5M2GYJeiOU05PjGauJ3HkZNHheU6WqsDqk6WcWgImMRph547p9Vu3pNM7qo87Um2AAovV/IGETkecuJnTE6+QRs2M0GroZDVNoM2klArWcYW4vWzXXuU99tj145ZxrnzoZDG7cshXl6P5PdD8oCvgtyFBooe4efvPhF/fHozWorVOPfhkjjMoclhoxnzCAaS2lvU/kPZ0qOMeu4SzTPU3OXHV+Lc4ztVA45BGzYP42Aox9hftzzzs2eBaLOpVh2B5qKZkGfYNtexGHneCuKVtfuba0+Rl0vKfeORNDd8bA1Vj3DEpJG49sw5fr31hSN3By1UGRnlZeMRFzpeME0/EIAnYRkRmjz7sg6FKWyoeu7miM1MDdtIPskG4faYpSqyO22eOpur8lQMomXEAdXwUM2B8yd0he2q00VEdxw1dRTefHw83XhSOLxH8rBK0eGI55rR15s3HLk7GME6FNJgyrgYUZIE0wFV3UQllSwj677SeO7xfO4W0TJDmFuGh/h2o7qe1587D6fM6dbWJY+W0bcvzlDljkxspzKgmlyvrsak36FU8KTkrluso2KbvcyXBY7cHbRgBgRMEjkkIFHdjVzOEAqZRpap2GUhy4SEYR8to/LgdWgYWSbhewC9l1wtYwu+Tn5Kv+7y8NdOPaCqbyvclmBfwSO0SCKIVGQea8tp7g6NgDSaOx99onsgB4J87sb1Vj9LPfeEAVUwu9wyacg2tkiHRb6dhiF3wWR15ydfCLu7syUhFFJ/nnx0D69t6/pYj5PQVOVkHvlbT5gZm8Gb1JkXPTLy3EXUy2MP4MjdQQu7GarVQlVZRuO5W0bL8PHaciJM57mromXSyCRKOcbgHOsV/5yEmOeuMJ0E/fyyJVMwrqMFr18ylZNJ7MdG+rm1cfkFMHS/R1VCMx9QBYD5E7vw5KcuiER18eXedlKc/AsKck+KdnKhkA6Z8NK2fdjbN4AjJ4/KpT6TAcHgZpXJMjqYpvyV1Skb9AoeHlXbqlBIlWeaynMPQiGDpGg2skyjkLtwPVSdLx/P/snXHokrT5pZ3aepP+k0B7gptXwOdt1h/NR+m5WzZHXzncPHX7MA7zxlFk793N3htqJKlrGYeVsPOHJvMpz++XsAAGtvujiX+mwGBG0HVAdtmE8wQjegqqpQlVsmKVrGJgd80HmkCYnMc8HrLBAnPak6Sz7eW/w59DynZ7mBiOeerKUHtgTtquQPU3Lly1VWlYru55fM4+FkGYdhBRNVQbzPGadt616ls3nuGs1dU6F0OTnGpG8msreD9pYC5ozvVNZfzZdjMUjho1FkGZHMVWdSydtegWi56lwuOHKigecul2V0x/GdvY3mLoNIwuK5qEjcWJZplAFVIvouEW0molXctk8Q0Xoietz/dxG378NE9BwRrSGi82tluIM9Xt3Vix37DlodYxItI5YFwC2Npoat5l5O9Nz1mvtL2/cpZBl5eZkss/qT5+PO95+WaGOalZjSyDJvXCZPypYF4lyApzfulpbjZRmTjunWa07CeUdOTPRfBwYVsozmwKqeLR/kBfSeO3+IeLh4XPask/VhdxPP/fsALpBs/zJjbLH/73cAQEQLUFlb9Uj/mG8EC2Y7DD1O+Je7sOTTd9odZDCgWn0Y+AFVcV8c9rllqp9lD1jSGqofuXWVfJ8iikYaF62IEAkQnJMYLVOrUMibLluET7xmgfVxOojX6NnNe6XlKubKZZkA0bBGfdkAh3FvRsYDqtxopfIyao7nTzkmMQlkrErHkJztssFkGcbYfQBMF7m+FMBPGWN9jLEXATwH4LgM9jkMMWzIqXqMabSMXZhlJM5dQbxB+yocHIjnv2VgxuSehOCc+HTG/YNlozemNLllPI+sF7BOgkn+dyCqR8eljOhfvozOcx3dXooEAxQNB1TDjgPq381ccyfhu36/KYbTDNX3EtFKX7YZ42+bAuAVrsw6f5vDMIVR4jCJHGIiQ8QWtjAsD+iTWemqk61epAv1q/w1fyoDUu/3pYUyY7jhlyvx1T8+l3hs2mgKk5S+NjBN+cMvWpHk7fJldJdzbHtL5Ltpbpm0M1TF42XlxN8/7XyEcHyiUTR3Bb4J4DAAiwFsBPBFf7vMbOmtQkRXEdEKIlqxZcuWlGY41BpM+GtSFjDT020X64gMqEqekGAAVDc+sPtAv6Re+YBqmmcwGAwM3hAYgNue2GB0bNpQyKwpfUUYe+7gvXFhn+RU+IlGpojkktcNqEZDXKRltJo7V3lS5I+qnqTfIUw/0MjkzhjbxBgbZIyVAXwbVellHYBpXNGpAKR3NmPsZsbYMsbYsp4edQIih6GFVSgkV8hEcrHlpEhWSMkDkjSgCgDbJPKIKhQyTfRKMBh50HT174ztAdHQwTxg2ll4nCwT83bDv3HStDnPVm79Vt1R1Y5DvRKTrt3WkjoqJ8mTD5B03aqx+I0zoBoDEU3ivr4eQBBJcxuA5UTUSkSzAMwF8FA2Ex2GEkzySUR12n8VsQWiJRBnc1oYo80KqasvWEQkUq1KljGzKoKAZ3nP3ZR6077u5+25m/4cfLSMeLF0nnt0m1hp9ePyY6dhRMl0QJWzSfPL/fzqE6Xb+UXATTX3ez94Bv50w5nh9sT7uM6ie+IkJiL6CYAzAHQT0ToAHwdwBhEtRuW+XQvgagBgjK0mop8DeBLAAIBrGWODtTHdoR4I7lcdfwS3LJ+qwCR00jZaZjDiuUvIPbBDU8fmPb2xbWli0lUY9GdXhuRuUXfaSUwDOZO7aWfBE2mco+MShIzbbnvvKbj76c344p3PxPYdNXWUVi7hwU9iUvWRG3YeUGax5BcBV51LgOB3mjGuI7I9cQF3/e7ckUjujLErJJu/oyn/WQCfzWKUQ+PAKHGYZCanSU4a26yQvCwje9Bl2SlF7Ngf19wZgEu+9idJff5+C4IeFDR3G95NG4XBx4XnAdMBWl6WUUkVFCkflK1u6+lqxTEzxkAFPoeLUSikptyB/kFlB8FHHMVkGKHTVZ1rcH++9ujJuPr02bH99Z7E5NIPOOhhMQknIsuEXrlmQNUycyJvgs5zNx0QlNUrr9EcAc8Gmns9skLW03MvehS2x2eFFE2XSXVhTDzvjSfYIkvQJUMwmE6K1UA+/4ZFuGDhROw/KBcSeHKPR/5EoRr4Du7nwyd1SXM7Vd8uGlhzd2g+/PCBl7D85r/GtptEy4i3Kr8ohsmAqrHnzpGOlAgJ2LHvIG5fudGswtAOuQFpnsGgrr5QljE/Nq3nnrfmrnsT4MmWOB6NR5jE5Rpp2CRBy/AtlpOYKrJMvNzly6ahq62kbCoqy4iau/57gOD+VJF/sNVySdrUcJ67AwDgo79eJd1uk/KX7wFMvOcsuWVk3o9HhD89t9WwtipU7ad5BmOhkPWQZTTkTmRnA6DvLCqhiYNh3SqTg04gEu0ifdsirSffWjTLLROVZWIMaicAACAASURBVNTlVD/qCM1EsBi5K1zi4LIlpRyuV95+R+4OWlRnqKofeJl3byLj2K6hytugcNxTLeasS2lb+WteZ7lcWajkYAodPPUkprK6rQIRBizZXddZiBq4avWrNy2bhg07D2Dh5FG49seP+uURK5t0aU0192pqg3gWRx6qSBpxlu/PrjoBG3f1Sm1Ueu4JmVCDtutF7k6WcdDia398Dn95bqvevZaENNpMYjL13ZMmMRGle3DUoZBpNHeG/sFqOoMyY8YDsmkf+n5NnHuQ0mDiyDbj+rTkLsgkYqRUWK7o4YYLDkdnWzFSHhA7aT0ZizKQCroZpqpyPERyP372OLxuiXxyvar+4Bqofsd6e+6O3B20+PGDL+PN//lgihmqyZJLIMts3XsQX//js8n189Eykv0eUa7kngaD5ajXXgkLNUOa3DJBmyoE+u+s7g7ccMHhmeuLkG34n520FfHcFWXC9grqEMWzDx8ffuZnv16wcKKyPhUxv/+ceRorolCFrA5yA80yVGeoOnJ3aCDoNPTQ/45o7sl18iF3X/hDPM6Zx5GTRyZq7kA6glRJTmmewUHGIsnJrKJlaqC5B51dwSO8+bjpWDJ9dKo2ApSEXC+B724yKK3LwV/9Ht0Q9dyj+276u0Xh50JInBWd/7Klcq9bdYVHtZdwy3tOwmVLpqCrLapWtxY9XH5MNbWycoaqfw1U92B1QNWRu0MDwSj9ABcTb5K2wDTI4/wjJ2BkWymaz11x56bR3FV2pCL3MsNPH345/G7zVpD2mb/mjMOUIYPB5fA8wqj2Er5z5bHpGvERIVuPs1lxnrwXKzs9Xf71WHsx7Zv7HIZC6qG7xsfMGIMvvWlxjJyJCJ+//GgsmloJb1SRs8mi8ICTZRwaDAaSewRiPnMZxEUhVCgWPBBFy8tefRlLq7krvM4wV405Q5cZw+fuWAOg4uVu3tNnTPBFz8PCKSON2wowbWw7fn3NydJ9wfXg1xjNgqIXlUnCQVLV24/ElrKgy+gIt1UzoBrpOETJI8dxlABJskswrq3aH3r2znN3aCQYTWKSyDK6w0xnQpa8SlRG0iQmIEo+pshxDlN0/U9LWzwPuP1/n2rfqAahLBNONspGLHznycsy6rTJ1fIlyetW0oShmMbPgfewRW86z980wAfPn4+RbUXM7umQ7g/uZ5VnH8bB14l1Hbk7YP/BeI5zEVrPXVjQ484nNxnFuRt7tIHnnpB+gIGlenDyTBwWyTlv+RaRxatUec6xWZEZncaIFMJFuigHVLnyQW52mwFVXVqAqC3V+nTIooicMX88Vn7ifHS0yiPIywmae0j+KRyQNHDk7oCP/Xp1ciENEYcPq/93/c4DWL/jQNJhxggG8fi61Otk5ifL2GLSqLZIpEnBcipiLd7WqwOq+bRBESmkWp9yQJX7HES+8B2RGJcu2tfRwhFpTHPn3iKEJfzUcxdqJ4kETao6EOe5O9Qdr+zYn1jGaIFsrkxvihmaKhQ9Dx5RhDjT5OxWQe112oWuFQsUrsAE2HvuWbxKldfvCXJMVllGXBMjKVkb78UG1yMqr+nb42Wgs7jQx8qxJPkcdwR41FLtLido6kmafd5w5O5graer9kXGR0V3PgOKBYpNo5fpmnyUjg3ykmWKnhfmlAm+2yH/hz4cUOXWGM2jPkCYxKQoLxtQje5PEmaqWDp9DNbedHH1WIksE0A9BmDUVCoE5K1qYzD07B25O9QJZqssqfeJi0ID+cgxAYpehQLE13kRadvMK3FYwaNIjLtundc82jNBNVomKlukhTK00eAayhYw1+WnEVESrqdWllHUUctVkMqhpq4fUE0TrpsGjtyHMbbs6cMBRQpTG5iQom0aXZNl9kxRGVA1k2XSNJef507o7R+MfLdBLR75MM49/JutFb5TLXhcbhmD8gHE602Rz2r7poweEfkenFN3Z0tsfdakfEG1QBAKmZQS2MW5OyTi2M/+D97wrb9krscossXy+HAJ0RzIvRIKmZxbhjGWqj31DNXkh3D1J88PPxcLFJFlbFGLwb5CTnJMWJ/giYcDqoo5C/J29VFPKvR0tUa+e0RY9cnzcf+HzpKnE5bZU0NePXbWWADxFZoCVGWb+pC7ywo5zLF6w+7MdWSVZWT7BlIsVqFCseBBpAmZnB0NyFSj4EXfAvb0ykNBk7xAAJGwuILnRVIBHMxA9LYY01ECAHS1FrGnr3o+YihknrIMgGTNPaHBpBmqADC6vYSd+/tj5YiATv/6h+kHEgdUa0es7zh5Js5bMAHTxrZL9zecLENE3yWizUS0itv2eSJ6mohWEtGtRDTa3z6TiA4Q0eP+v2/V0niHfGA2CKmZaSo5Ps/VgYIBVR5SQmBmHZX4Wrz85gfkBS2fwZJQ70FNtkYd0hDwpFEj8D/Xn46PXbIgsr3g5UPqAeIEmxAtI/uZ+Dh3Sr7Md3/gDDzw4bO1tsTuhyEYUCUiJbED1WcibYI4W5jIMt8HcIGw7U4ACxljiwA8A+DD3L7nGWOL/X/vzsdMh1rCiNp1nrtkWxASmI8s40lWoDe3RYSp52Tr5YmdxsGBdOMhaR/9OeM7Y4O4VVkmn1DIyaOjqYOTBjBl7UUmoxm0OaajBRNH6VMWmw4Y14dW5QicqIZJHMYYuw/AdmHbHxhjwbvfAwCmxg50qCv+8txWzLzxt6mO5QlY5cVrZ6hKdobknsqiKIoFihGtjDTaW9Wr6fBQaaIqmGqk4vR62wU7glayaLKq/CvB5lLBw6+vPRkfvfgI67p//K7jcfq8nsi2pPw7I1r0v0nSJCZTiJO0soyj1ArV3DL1aS+PAdV3APhv7vssInqMiO4lImWiDCK6iohWENGKLVu25GDGoY3frbJbN5QH70mpPG295i6RZQbNomVMnrWWome0Gs7McR1Gbwr/eNacWFhdkm3//b5Tcc//OUNbPu6525F7Hh2heFlkA6qLp43GyBEl67pPmtMtyZhY+au67l2SqfriJKY8dPDqmq32MlG9EPT1tjOX0yITuRPRRwAMAPiRv2kjgOmMsSUArgfwYyKSprljjN3MGFvGGFvW09MjK+JQJ/DyuIpgTJbZ49Fv6LV2KvJ08GgpxMldxEg/B7fJ+EFL0cM5R0xILMc3ecSkkZjZrff4+UWWAfOUxqp233PGYekq4KBKGJaWXkRJqzqgKj/ZzjYJufPH5+RJm4YXDqnn7sdK1kuWSR0tQ0RXArgEwNnMf6IYY30A+vzPjxDR8wDmAViRg60ONQKLeO4MskdfR1S6AdWkaJnO1qIyWiVAxXPXyzJEBAYz77fgkZGHb/sMtrfYP06tRS8WPhm0e7SfP9wGseukyCmTVnsXSTTJc5ctPB2Lc8+B60KzEuypBX51zUlG17Oz1Y9oarN/a0qDVORORBcAuAHA6Yyx/dz2HgDbGWODRDQbwFwAL+RiqUPNwEw8d91KTNJQSKav0Icqwx6P1qIX627E5EsewThaxjQtgK1ckKQvixjf1YqJo9qwct0upQW2iF8n+UBjam1b0cmmDYXMC7FJTLnOkdZj6fQxRuWuPn02OloLeOOy+gxRJj5ZRPQTAGcA6CaidQA+jkp0TCuAO/0f7wE/MuY0AJ8iogEAgwDezRjbLq3YoWHAPwipJgFJjjloOKBqJMsYe+4sPJeWgqcc0DR/hTcqFkLmpaapP0i2kEYfFutsK1ZsGhDCMtNybiylgv/VZgZzLYg3ltq4AdFWKuBdp86uW3uJTxZj7ArJ5u8oyt4C4JasRjnYQ3y2fv3YemzYdQDXnDEn8dio5q6IlrEcUDWFmeZeiHmksXjr0JjKn1PmduOPT2+W1mea88VelrEkd5Vn7m9OI52IdQZvRr2C9JNelom+9ZT871ZvORpZhoiqkS8WdYr53INb8rKlU3DhwknmtjUR3AzVOuPpV3fjlkfW4Z8uOqKmXsZ1P3scAIzInZlEy2gnManrvvWx9fjYJQswtqNFun9cp3w7j5Zicpw7UVRH113ZgkdW3qNp55VGc9fZmWZNB/GWCsi9r9885n52dwde2LpPuk+UZa4+fTZ2HjiIK0+aoazvC5cfHVl0WryaIol/8LzDQSDlItcyiFE8QRvnLZiIcxckD543I1xumTrjzd9+EN++/0Xs2N8/1KZgw84D+JffPRWZiq+CfhKT/vjP/36Nct+4jlblvgAyWSY+QOin/A33q2mz6BHMgnnsOl/Rc//K8sVWx4utpgkRFDu9Tj/238Zz1/kcoqTV1VbCZ153lLZje8MxU3H+kRPVlQoY1V7Cp1+3MLIKUxJUk5gaWKWpORy51xl5rfqTB6772eP4j/tewNpt1cU60sW552yYgJZCfEA1rrlHbdHp1QWPjDTioE7TNyxxQHXp9DH4h1NnSctevGgSvvmWpVL2qbYb3T47IRTTPyryLSBd0XPn6778mOgAn474bdMYyyA+AxFZJmWd8ayQKStqIjhZZojQCCQvi0VXau6aetLGc49sKxrJI0YDqogOqOoIyiMyyn1jSzSi586nxBXx729emlhfmtSwYnOBTaLnzr8VfP7yo7Fqw248tbGShC7p2mVFTJbJwbuuzlCtxssAQ5tuYKjhPPc6I7j5tuztwy9WvDLE1sQR9DmxPEzazigdu3uG8eaySUyyxZIZFwqp06srS/Yl6zJZB1QLXnLGQ2m7oMhfu2PlNomeu24MQ5fYKo+MhrXwa0jw3MXthyIcuQ8R3vG9h/HBX67Elj190v27e/vxq0fX1dkqXZy7+pisMzGTUCrGaU42oAqYae4ewWicwZZcRwlT+j2ixHBGfvckPzmW0nQDc8TzDnRr1UQp2XeP4qseBchjoYmkAdU0EDv7BngxHnI4WWaIsMkndZX2+4nbVuNXj67HzO4Oo0kSed3L6sRhuklMtX2SSjLPXUIykWntmvrIlNwtOWd8VzRzoU6W4csAlcFXcQAxjdMpHjJC4bmLJcXFpttKBfQPxmcO11pzT4twUlsw9hL9ekjCee51hunNtvtA5cHavLs3dxvWvLoHL3ODqDxUD0UtPHePyKhjKBW8RJL0vECWSdbcATPN3RaxpF0GnrsszwgJf8XtNjb0dFaika4+PZqnJvbmI+wTO5q/P3GG0l5bxD137rNB9a89ejLGtMffkip1Hcp0HoXz3IcISaQWvOIHJJ8nzv+3+wAAS6aPltglP0aruCecy+oNu7DrQH9MtjDlifaWQqLmLi6hrSNVc1nGDkSEBZNG4slgYNJL1ny92EBgNp1YPHRESwFrb7pYaqvqu+dRJAlawSN86tKF4eehxlevWBLbJq40FdyTh7Dk7jz3oUIStQREuOtAnePhaxAKuXLdLrz1Ow9K9lDidfjc3y3y86RHn1Ixt4wQJJEQx02Gsoyv4xvKTh4Bv3tfNcu1jSwT8V5jH4AJI1uNSF/0XFWerHZAlShMWyDaZpqXRwv/cn7NJ+lcZBlFJYcyuTvPvc4wvdlsyT0v2VsdCqnR3A3qlSXHIjK3O05Gcc24Mokp8Ng05A4zz90WsgVFEmWZYPq+pByB8NxnL/T3U/jGpbVBM1CqL8dr7tGY/chgay7cLnrVeQyoRmty46nOc29YBCFse/vyl2V0qIZCmkcfpB1QjQopmkLQk1G1LrNJTMZx7mHbZuQT09w9ig38ju+KzsgthKTEleM+FgseigWvElZpZIN6oDRSLubhR/dFPffq3jw89/Aek5xRWk87uM7iZLZDWYN3nvsQIYkPgyW56j3ZSRkKqTkmdSikSWhf+FckLUVBg7qNo2UsiUEsXaD4wt73fvBM9HMx9rKUvOE55xAto+JinYdPVJk4Jqs0T81dNRM3DZQDqocutzvPvf5Qe0w8AvJReY22pG9aXhkKqcvnzlF/iyiGa0Awm8QESMhI2B9E3lQ9d53mXptQSNEoT6K5j2gpYCS3WIPWTsvmAdl1UmnR6gHVgkc4ftZYqR25xLmLWSEz18jJMoHnnlB+5rj2HFptbDjPvUFRTiAfU1JkjMUyJmrL+3/FB840FLLgUSWTvwGIkh/CgHRiGrtkuTcGs0lMxgOqiSXE8pUjjp46Ck/4YwxJXBjEjfPzHaqRH/a0l/iGo9h+1JRReOjF7f4+wjVnzEH/IMNX7no2NsEpK6opIrLXFSCWFTKIlpGUvf9DZ2JUe31WQxpKOHKvM0yf18GEUK44NcnJqswquq6pn/9/fvEE7lmzJTbNXOv5c7uKBQIUY8CxySuGNsmgThyWTBwEYMAg/YAtAhv+37uOx8advb4d+rMM9vOdTRaZImlsItwuXP0bLzwco0aU8KU7nwFRhSyPmNQVK5vHdP7qbRDvxNJq5OqskPH6po1tfq8dcOQ+5FBRZrLnbkbXg2XmrxlqVv6eNVvk7WmO4b1OXe6R+NqZyW8UKv1ZHS0j3y+WrYUsExQf2VbCyIklvw59JYHMIZupbBr5IrMhgOrnEOsqFTwc5a/ZWr12csLMiqyTmGSonuchLLILSBRIiei7RLSZiFZx28YS0Z1E9Kz/dwy378NE9BwRrSGi82tleLNARW7VAVXFcYb1B6RhO+YpPmSGjntspZ5k6C0L7JAlCpPVZBItY665Ww6oSsonh0IGnjtXj/DXzohkmyrb49sCh0LUr/OmS1VyuiwoCJ57NVrm0IXJk/h9ABcI224EcBdjbC6Au/zvIKIFAJYDONI/5htEZLf2WJMjpmUryC142FW5Z97740ej9Si4qpzQSZgiF89d+G4T5y4bQBW/M4luLa2LYJXy1/StR9Zi4iQmf3+5nGy7iWRhqrnL6gpMECdW5Z9ZMaqH51G9Oitk9rqHKxLJnTF2HwBxketLAfzA//wDAK/jtv+UMdbHGHsRwHMAjsvJ1qZEEimryP33qzcZ1R94qFkXJdZGy4gDqoZ1GMkMghdZ3SEvFxKHVnM3k2Vs3T4ZkRt77gayjJkNyTap6g7uNXFAt1b8WK0/ewvifVeLRbiHG9KGQk5gjG0EAP/veH/7FAB8kvJ1/rYYiOgqIlpBRCu2bJHrvIcCVLdgQD5ZJ1IG44a2nnuWB840CsIkFDLMbZ4wOUd8Hc8lFFLRtuYASVsJA6qhLMN57ooKzTpDs2Nk1yd46woWLa+VjF2LqRuilBSsQFWyCM1tNuR95rLbQPpTMsZuZowtY4wt6+npydmMoUNv/yCuuPkBPLlht3R/XMtWyTL5TGKSeYRpYBOPriJWqSxj6GHFZ39Gv8cHVNV11WxAVVLeWJaJSEp27epssNHcz5w/HtefOw+feM2RkTJ5rL7EQwy3lU3gsoX4FvCvf7cIHzx/fiRe/1BDWnLfRESTAMD/u9nfvg7ANK7cVAAb0ps3/PDEKzvx1xe24eO3rUouDANZJmPEXl6au8mao0BlZqMpGdg8yLK8LbK6jDx31EaLlWvu+mMCOWFgkL++0cHBLDaYxrkDlbeIfzx7bhgDrpLEbrrsKPzi3SfaG+ejFhkbxZm+YztacO2Zc2owXjB8kJbcbwNwpf/5SgC/4bYvJ6JWIpoFYC6Ah7KZODyRVUesyjKmM0vl25NCKpUQ3zAMD2tvKSinvIs2Bt52GsQeWiL86bmt+O3fNsj3R4vi1mtOxttOmqlvI6tNMI9zl4ZCWrZfscG0ffPBWbGO5cdNx7Ezs3vEefJuA2QibjiYhEL+BMBfAcwnonVE9E4ANwE4l4ieBXCu/x2MsdUAfg7gSQB3ALiWMWY4X7E5EDyiD63djhVrxXHoOOknD6hmsycMqZTQs83zYErC7aWiRpYRA93TZ+9Tpaz93d9erVStG1AlwsIpo/CB8+Zp20gb525TRzBDNa9JTKY6vU0befOmTJ7LimpuGYcAiZOYGGNXKHadrSj/WQCfzWJUrfDzh1/BEZNGhpM1ao03fOuv0oUSeKhDIaOa+x+f3gQC4czDx0vLq1Ctx+qw2BNoqo2PaCkoyf1rdz0Xbyb1i4VclgmQFOdeKaOnAuvEYSk093CGKq+5h/WJRG3gbYsvNIpzMJLOFLJMVoiZRyPnlbIxT1bXIY5Daij5Q7esxGu+/iejsv2DZXzjnufQG1t7Uo/E6I/YgKq8XODIBR78O76/Am///sNWtgBctIzlcSKZG3vuklWTAnz97ii5mzyG8dzfFagWyA6gWw4uIIJEcrf23CWyTMITFkSoyOSzVLKM2L7Kc7eqq7YDqnkgjzzzzQZ3SRT46UMv43N3rMG37n3e6jjb+FpV6XJOoZBpUwen9ahHtBSMMwcSUep4ZDFRVBLZR/aFZfRt2Gvu8W3Ja7/6A6oGsoyJPbLJXSblpO0Jsz7zQp45hgKocsscynDkrsD+gxWPfV+NF8tQhkL6201DGVUkmTb9gFjetHNoMVjMOgDJGoq1K98uX0NVvT9S1lCWsWUKWfHE3DKyGapZomUMB1QtVJncdez3+It1Hz11tLEtScg7XLMZ4MhdAZN7pbd/EHesejW6MXFSjlnxvOLcyyk1d7Fd0+OLheSl5QJkeR6TFutIinNPKpMGUlkmoY0gXl86QzXNJKbY1N3kY5R11UhzP2lON9bedDHGdLRU6s+h+6gus+dIPoAj9wToSO1Ttz+Jd//wETz28o7c688rzj0kDUtyF+WgJHkokGI27uw1HtTyKHmB7KT2AsS43sCE5AFVO6SRZc46fAIA4JwjJsTqMR2f0dmgjnM3kGWCN4g6EmbalmQrWh3qcOSugMkNvXbrPgDAvr7qoKs9WemjZWTxz7v2my2azdeTObdMwvHnLfBJasEE7WBmrN4ExlLt7miNBnqZzswEzPXZk+d0A6jM3DSBrLokz33B5JFYe9PFWDI9TKya70BjFlmmRp57GluS4EIh43D53BOgo56AOG1G6k0Xnq5mhYzvu+LbDxi3lza3jIgXtuzT7h/T0YLn//kiFDzC1+56NltjBugUyT2F5p70hrFo6qjEUNZovfH68gzNS1NVI2ruqnYy1VGnjmg4wXnuCuhukue37AVjLPSqeU9VJNFNu3uxVzMoq4yW0US5PLkxnrcmMeWv0gIz3PZEchaJ6quxuSyTFiK5x+tW76vV8y/33O1byzNKRVWHXZx7HWWZlE3luXB3s8CRewJE0nzkpR04+4v34ocPvBR67vyNJcoXx//zXbjkq/cb1x/ANv2ACmlDIW2R5tEiSu50VPuTZRl1ncbRPJYnJdfc7eqwrT+pjCq/volZddPaSfrRCm4SUxyO3C3x3OY9AICV63YhyPXEx1zLOHTttv3K+pJCGLOn/M3Hcy96hKM1M3vThu2l7XNKBX0kiY6YTG21n6EqiZbJwO6q9j//hkXGdag8WpvZrjXX3HPoREJbM9fUPHDknoC+gegM1X6f0UtFLyROmwFEEXl47jqvvLe/rG3HFIOMYVZ3R7ZKMiAeDSNGy5iHDZp6d3mQWpp7Q0WqwTl2tZUs6spBcx8GjFlw7B6DI3cFgofiRw++HIlO6fdHOkteNSe4zc1vGt5mkxNGl5t8zabKm4bsDcGG7xkD2jU6dxrvyyQUstZyUq2RRgvOMrfK9HcwkmXCCJTaMqabxFQbOHI3wLZ9feHnIO92seBJpZOsOVwC6JbZEwlDJ908HQy+5sCRHS35LodLMCfvWCZB8XsNnu08qkw1oKqYoWoys9a0ObP0A3Z1pkUu19mrT0c0nODIXQH+FuFfbfv92MJSwZNKJ3nlcNHJMnFyV0eh7/cTn+Xh/4qDmDzSEkBau4iAP7z/NM3+PHTc7HUUC+nrUGd0TF1lte4GCoWMtJnymovL7Dk4cjcCf7/0D/iae4GkkShv+54+c6OxLBNmhYzvE3VcXX+SNv2ADEnhhwGMBywNCqoyCBII8yZ0qes2M0GLIfPcM8gyprDz3Gsty2Sv301iisORuwL8/cZ/HvA996Ln5ZK5UeVzM00IoxjepksuNpDTDFUgwXNPUR8RUrvuSTNSaxkjboNUmrui/er2elGYXB6qTSvZ4NIPxHHIkHvWgbmzvngPbn1sXRgtU+Q899RL2SHdSkxieJ0uoib1Yh0SjCjlr7l3tSW8DRjaHffssyMXWSZTKKTddsDmrcm8zHDgS0fqcaQmdyKaT0SPc/92E9F1RPQJIlrPbb8oT4PTwpbc+HulzCrT76//+RMY8KNlymUWTu03TctbqVeQVBTlymH6AUm2QFHa0SQXG8wpzh2oLH6tQhoiJCJ89JIFWDxtdBazGhpZZqhKduj3W8AscViCPTkh39wyjuUDpCZ3xtgaxthixthiAMcA2A/gVn/3l4N9jLHf5WFoVpiQ22CZ4R/+a0Vs7dMv/mFNpQ5WDYXsL7NcvGLVG4XOcxdv30HGDEIqJZ2EsZUVtBTMbhfzSUIVHf/q02ZbWqKOJKl+b4yHPE9ZRtwv32cYCmnkuddHx+ZtTttWIez4cjCoSZCXLHM2gOcZYy/lVF/uMJFltu3tw51PbsK7f/hohBxuX7kx/HzQJ/eBwXJVlsnA7qojw0y9Us/dTJYpFSgcI8hDltF57mlgEmqnGitIWkO1UR7ybDlP9OcoPcK4Y7Xx3M3qHEoMJwmpXsjraV0O4Cfc9/cS0Uoi+i4RjZEdQERXEdEKIlqxZcuWnMxQw0QWDx7EwXJZeUMf9KNlBsss1YCqabSMLs5dVVZEqeAp88GneWD1sgz32XgiTfpgu8QBVcN6PnbJAuu2bZCK3MNDFB1bLjKGgRkUlG38EVUiwj+eNQfnLpiYvbImQWZyJ6IWAK8F8At/0zcBHAZgMYCNAL4oO44xdjNjbBljbFlPT09WMxIheoB3r9mM3zy+PrItXIlew9aB594/yHLx3FUPcFDnoIScxWfhtsc3SD3colf13EWMGmE+hT1AS9HDdefMxYULc3qA9NKydlsiHxgS0jtPmWVUTocfvvN4XKWQllKlH8hqUMM0Yoa8+o7rz5uPozT5jw415OG5XwjgUcbYJgBgjG1ijA0yn51XmAAAHiVJREFUxsoAvg3guBzayAyRf9/+vYfxvp8+Hi3j/x0sM+W93z/gyzLlci5L4akO1ckyIj7z26ekfQQ/0UqsJk2ET0vBw3XnzMNp8+KdcUQ3tdDcZeX5r9XrIJRJ8NTryV2nzO3GP110hHRfIcMkJtVPn8+ydCayTH4DuPp2qvjUpQtr2tahhDzI/QpwkgwRTeL2vR7AqhzaqAtCb1lDqMGA6gAny8i8axXExyRo6e41m/HIS9Xl+mxkGb4eHny4pujZ7+4dwMvbDxjVHSCQZQZsTliDPHXSRtWF0yUOqxyj+uX1uWUM2zCyw67OPLBg8sg6ttbcyLQSExG1AzgXwNXc5s8R0WJU7s21wr4hg412bSLLDAyWw4cv04Cqf+jb/Zmtwco/gQmmDrbMhqLnYWCQYevePvzq0fWx/Vv39sW26RBEywxIjEpDrirPkIgSR4CTmmsUsrdZpSuAMr49x3Mymh3M8m83rS0O9shE7oyx/QDGCdvemsmiGsGEf4MyMvIKEHrug9UyutmkIuLL7CWFQpqxu8zkUqGSufLd/+8RrHgp/SLeAaqeex5R82rPMCLLCGXFY+VHNU68czENu/tQyzLZYTKgGrztDYPxVAcJDp0ZqiZleH1XcUc/8EIlBp7vAMr8cTnZpdKalfVICpYKHgYZw6u7e+0NkyAg937JIC0pPuuQMFcn4VjR27evox7Iks9dlNLyPCWTzi/03B39DkscOuSuYMnNHPHZyCt8FEqWdUrThELK+EJWT5EbUM0DpYLac88my8i3Z0Gj0FE6WcbX3JWiu+ZY08FsA7tUb015o1E64mbDIUPuKo477p/vwkvb9vllqoWS7rd+juBsVk2KD6jayzKzuztj22T1BLJMXg9Pa1GtuSfhs6+PR0EYTWJKm1umQQgjjSyjfqMxGgY1a8OgTOAQ1fpSujeD2qApyX1Pbz9eFtct1ZDE6Z+/p1LEgrP4cMnguFQDq8IhBw4O+nX5dUoCU3pGtsa2ycoVudWi8kA4oCqJlkkiHl0CrdjDneYtoEH5IYPkXtNQSJt0yw17cR20aDpy377vIM790n047fN3R7a/tH1f4rERzz3hfu4flMgyOWju1/zoEb8uzSCtdJMkWqbgpfKyZSh6FGajTFOnLK6aFCOq0QFVs7bi6Qgag5Ayra+bQnM3bc5oQDXU3GsL13fUBk1H7ks/fWdsAPGFLXvx2q//OfFYnrO27T2oLctzrtWAqnAji8f86bmtkTplPCp7Q1BFy2zZ06dMQWCDEpc07Oip8SyOkQFVydMqm4Zf02e6QQgj3Rqq+mN0u8e0t5i1YSbMJLbn0LhoOnKX4dVdZtEivJf8pTuf0ZblCdY2bDHSZsJiHbI6pc68ZGNff4XV1++0m6wkA59X5uJFk0L93RRScleFQuYgyzQKH2WJ4Y7Pyk0+ZmxHCx78p7MTy5nUVS/P3aE2aHpyPzhQNn7AbNSGCLlnyJmujpZR2yTrEPoG4u757t7+FBbJISYN6+6M6v69/lqtKug893gqgep34wHVWCjk8KWkxAlaCfsnjGwDoF8W0Yjcw7I1Tj8wfH+qhkamSUyNjpe37cdpn78bb1w21ai8TY6YMqve9FUiThMtI+4P6lRr7jLC75cMcualtwPxXO4imR/gvsueVZn2XMt1L4czX+RBdvd/6Ew9uRtcocBpyWNBbh0aZXyk2dDU5P7Mpj0AgN+v3mRU3oYLmUSW0a2IZFJPdHvlr0jQM2/8rbT8QYnnnmX5PxGiDDN/Yhf+8vy28Dv/gB6QePHi0oCAWSikrP40+4cjYrKMxTlOG9uu3W82Q9W+XYfGQVPLMgHpmg5qmWrms7s7Ih2BTh8XEUs/ECsQtWVf34DRG8VBiefen1OaACA6oAoA33zLMfjEa+S50GVrrcpWcZoyeoT0eF52SnsGw/lVXzVDVUR7S/o1bW1yy9Sa24fzb9XIaGrPPSBI08UGTFWZ7s7WyBT8MFrGyrqgUbmHHdg+UGZST1jEqvW7Y9tkhJ8WouY+akQJJxw2Tlr2e28/Fq/u6sWbbn4g3FYUUt92tBTwYT9NrtFU+ISrGwsbHMaEYXI9fnbVCZg+Tu+d69tIRphbJnUrZhjGP1VDo8nJPfgUffDf9YOHFeXN6NnzgMfW7owdpzv+FytewWd++xTGdUZD1RgY/vWOpyW2VAh014F+7DqQbmBUJtWkhUjOQFRH5899xrgOzBjXES0rvD2dv3Ai2nwPP48UtuKlr/nqQTVEt3+PiG9LARiAE2bLO1ZTBJdnfFd8QlykIQzvjvJQRtOQ+4MvbMMHf7kysq2aFiBa9n+e2iytw9RzF6eUm6Qf+NhvVqG3v4yutuglZwz4j/teCL9XZ70yjGmvkPvuAwNmhgnoG0j2+E0hGxClCLnrj1cRFaAn8ENR9/3ymxbjjlWv4ohJQm7zHC8BEeEryxfjmBnSVTABACX/bS3Nql22tjjkj6bR3D/92yfx8vZoyoEguZd52lyzcqIXGx6mOVxFfh8SOiQKNXdglD8hJa3nLtO+00I2IMp740njAjpyz4O0klZqGkpcd85cq/Kj21uw/Ljpyv15LHYOAJcunoKpY9TSzrIZY/BPFx2Of/27Rfk0qEDj/FLNhaYh9/0H415qMKBo+jCYkrvoxVZlGfUx1XQC0e3b9sVnwt6+cgMGywyjfY9p5379bFkV/vPKY1MdJ4MsNwy/Kena6XLLaGG8EpW83LSxI/CZ1w3t0m2XLJqUXMgA9SZBIsJVpx2G0YazXtO3U9PqD1k0DbkfkJB7kJo2y4IXMog3o0mcezVFgb4RAuG9P34MQHUSismAqgwzxrVjbIf+wTTlXFnEkWchy8g0+wAmkksSAcTDBis4YuJIvOWEGYn11xaOvRzqj0zkTkRriehvRPQ4Ea3wt40lojuJ6Fn/r1rUyxEyAgw0ZxNuv//ZLUgbeGeSzz0osyEhFQJ/HkFsuWz2qQxtJQ+XLZ0SfveIEgcWTQcepeTu8eSuv3a6dsRd8vQKevtik8EaiE/ztsU0mdpwQSNJaM2EPDz3Mxljixljy/zvNwK4izE2F8Bd/veaQ+a59/aba+5v/c5DVpOYeAShjLpJQ2l00iD80DTq5T2nz8Hp83rC74RkOcSY3GXJwIjX3PXH68wwGVBNQpYJP/VCmiRiPBwHOtigFtEylwI4w//8AwD3ALihBu1EIPNugynyxpq7IbuL9f304VcwfmRr7tpkqyW5i7HoHlEioXgeAAPVRzagygcNJXegOs89f9ZqJCIMTMmS/jeC5nLcHWqErOTOAPyBiBiA/2CM3QxgAmNsIwAwxjYS0XjZgUR0FYCrAGD6dHVkQBYEhJ+35i5i/c4DuOGWv6U7WIPQczecjCSSOyjZWzQlHPmAqrksk5XXbI9vIG4PO6/M16Chzsqh0ZFVljmZMbYUwIUAriWi00wPZIzdzBhbxhhb1tPTk3xACgSeuym5myYOq5fjZCvLtBS9iBfsKch9/oQurowZYUhDIS0GVMV2eKKKae7854R6rz93HpbNGAPxV2lEzz2viVXOcXcwQSZyZ4xt8P9uBnArgOMAbCKiSQDg/5XPGKoDggFV0xwrOebZygUFz5LcC1Hfjiju66296WK0lex/9iTPPTkKKAp+UNBoKryi+n88ey5++Z6Tcov9rgWCy5Q1u2IjdVgOjY/U5E5EHUTUFXwGcB6AVQBuA3ClX+xKAL/JamRayAZZdTCNQrBJDZwFgWecVpbxCHhha3x5wUHOftMzkabsjWju+uO1KQaEfWk4TGx+8bRKkNYVx9dG8rNB0MXm5rk3cEfm0DjI4rlPAPAnInoCwEMAfssYuwPATQDOJaJnAZzrfx8S7O2zm7bfaJ47UeWfuedeiBClSqN9a4q4b+mAqo3mrl3jVJ0pM+hIk+Pco+1PHNWGtTddjDPnS4d86gqb1MYm9Tg4mCD1gCpj7AUAR0u2bwOQvM5XTugbGMSaV/dI9z2zaW/i8RNHtoVrrqZZJq+WYAwoeZ5xnHtL0Yt0BCXFxKE3HTsdO/b346b/jicsU0EmyxQice76462Iiatr2cyxAIBT5/bga398zuQQLUa2DV06payhkA4ONhj2icM+ffuT+OEDL0v3iblmZIjkDk9B7qUC5Zo3nUeZMZQKZDWgGqzItHjaaBQ1+Vxsw/J0i20ABpq7hSwTOPJzx3di4ZRRAIDjZo3F6PYSdu7PtnTgEx8/L9PxWTCcM1U6DD8M+/QDK9buyHQ8z0llwwy5PI21FvNLzhVrhzEUCx7W7UjupICop56UNMyWZ6Seu1UoZPR4vj6VKSIZ6jok87VW1XU8/ekLzCqxRFWWceTuUD8Me3LPqqREVlRKcbxK+sgDZVbJpvjgi9uVZXjObS16IVHqcrmkgczr5LcNJugy4tEFzr48SC+Pd6e2HLNo8gjOLy9VptnSDzjUBsOe3LPr5NXjzePhq5+1qWwB/O+z5qSyytQePtc2P6CapO/aEmpybhn98WLnkDpL5DBEXnHubhKTgw2GPblnpnbec/e/fOHy2DixEknkPlmxTqgJyix5wY253ISkSihkhQDyJs+k+mw194JOllGsAKTjxnqFp6ZBYFlunnvjnqpDA2H4k3vGO533jgPv8yh/EE/ZJvc5NuVfQEdr+jFrxhj6+vUDAX9/YjWssWJLxbpEz93SFtmAKo/EaBnhe0Rzj81QbS72CnIWJV3DJDjJ3sEGw57cs4KnkeqC2gnHcB1CS4Ln3pFhhfrBMjOawBS0USoQBnwikUXKZCGHJM/ddkC1wM2AykNuaGRvNhiPyCsUsoFP1aGBMOzJvTfBs00CnwmyrJADdNB57tedMzfRsw/Q09UaW1jDdFJVMOO0peiFRJKULiD4KL75qMYIkvRi2zh3XWfR3lJ525k+VlwCbni6rlWnYXja7zA80QTknt8i0NXZkOYPoS5a5riZYxM9+wAjSgWcNrc7ss04m6Xfv7UWCuHqU/KVk6qfVVZ/4Lz5OHLyyNj2JK/TNreMrr4Z49px81uPwRffaDP2kS9x3vKek3KrK43T4OCQFcOe3NMuQRdAloHQxsPSDagSkbHnXmYsIlVU7GFYOCVOtNEyUc89WBRc5hmbdlqXLJoc25ZE7lni3EWNnQCcd+REdLWVYIoRGeQvGY6Zkd8CYixnz72RB48dGgfDmtwZY9nJnZ/EZKi589CRN5F6/79cdlTMDrGfKDPgv95xfKINY9orJJikuUc8dw3RyPqrRHKXqGN/eH81A7R4eCGFp63jxrYMk8l+c+3J+K93HJf6+CQEA6l82GoauElQDjYY1ukH+gbKmQfSmCRaJi/P3SMKV1Pi0VL0cMb8eA77ILKmreSht7+MQcYwtqMFHuk17V+++yT8+fmtKBaqmnswm/NX15yEy77xl9h56U5Rdv5pBlTnTejCd9+2DO0txdigad6hmm0ZPPejp43O0ZI4Znd34KMXH4HXHh1/I0oD57c7mGBYk3seentZ4rknZyCsftZpvUSViUUiBgbLMbIrM4aRvgwxolRAb3858jqvkz1mdndgZneHX3dUc186vSovZFoMO+FYlXlnHT4BALDrQDQnTDQXvJFZWiSlWxhKEBHeders7PXkYIvDoYNhLcsEkkwWL1CWOMzm9bdFIwcQ5LJMmUliu1m1bNDhBFKHdvKO8F2vuUdtU0FG5PsP6tMn2y6zF9XczaCzuZHJPXc4193BAMOb3P3FOLoypHGN5JYJZRn9MXyHoPfcCeM6WzC6Pa61ikcxsDCyJgjPLKfobHSa+0+vOsGqDh4bdvVqj0ki91gSMMMoIlPkPaDaiHCSu4MNhje5+557ZwZyv/q06utyGs1dF+pIVNHkj581VrIz+rWSJKyy8dwjJ+CcI8bj+vPmVXZaeGqDg+o49yMnczNv/XOUVX1A4qVv2HlA226StKKboWoa/XHz3y9T7pONbTQrmm0Gr0NtMKyfCI8Ih/V0YFxHKwBg5jhx0ksyujtb8b/PmuMPWqbR3NVRKWVt5Eq0EcYYLl08BcfNGovrz52H/7zyWEwaVclLw3vFpwqx8CIGDGdDyvZ+6y1LAQD7heUJR7YVceOFh2vrs5VltPYpfoDFmoHPJM/9kkWT8JXli7VlHByaCVnWUJ1GRHcT0VNEtJqI3udv/wQRrSeix/1/F+VnbhRHTBqJuz5wBk6eMw5AOimSqEI0ZQb88enKWt5Zo2WKwsLWsvpisgwDxnS04OdXn4ipY6KdFH9eorYser2yGaqXLp6MD5w7T2r/6fN6MGX0CPz3+07FBQsnAYiT+4/edQIOnyiPtw8mPGXJCpmHH3r+kRO1+7/+5qW4dPGUxHouPmoSPnj+/Bwsqh1cdkgHE2SJlhkA8AHG2KP+QtmPENGd/r4vM8a+kN08MwSTf9JGXQRhgwG5Jz06Ec+9KIks8QAMIlweTybLizq6znTeK05KRBZ67lyjX1m+RNJ+5e/o9hL+fONZkX3iwuK6vu5bbzkGp37ubuvUy/zbjHjo9YqOCADec8ZhePSl+AIt3Z2tOG1eD+57ZouVHSL+/X8tzXR8PdCMsszhE7vwuiXJna+DObKsoboRwEb/8x4iegrAkPw6Jd8LPHbm2NjSekkx4kD8ld4jwpUnzsBtT2zAjoRl3VolnnvQWQTpes08d7WR/K72BPnhtUdPxrfufT7Rk9Vhv0WIaTVHjVm5ALpVlU6fF58DEOCGC9TyULPP3Gxmf/2O605LLuRghVw0dyKaCWAJgAf9Te8lopVE9F0iks7jJqKriGgFEa3YsiWbtxV4qd1dLVh708ViO9pjT57TjZ6u1sg2jwifvHQh7rz+9MS2ByWE4oXkXlbaIG4yTRKW5LkvmDwSa2+6GIf1dGrLBa/2Mj58wzFTK235HYnJhKfklZjErJDNTFW1RZP3YQ45ITO5E1EngFsAXMcY2w3gmwAOA7AYFc/+i7LjGGM3M8aWMcaW9fSoPTUTBPrtoGShah2F/PnGs3BYTyd6OqPkTv5VMdHe9/TGI0tes7gyE3HR1NF+PfHjRLIz9TrziufWndrp83qw9qaLMcdfCKRPs0D3mPZKJsu3nzxT2554DaLRPPmwVbOTnks/4GCDTORORCVUiP1HjLFfAQBjbBNjbJAxVgbwbQC1S9rhI9DcA72ZzxMiex6CuPOJI9sAAN0Szx1Qdwy85ikj91PndGPtTRdjlj9rVNpJSCYxmUCUZbKnX9C05Xck+/vUMs2IlgLW3nQxrj79MG07sXzu3JhAXqQc6P4/fGdyPp7hjGbvxBzyQZZoGQLwHQBPMca+xG2fxBV7PYBV6c0zQ+i5C5N/AHlkwUcvXoBnP3thKA1M8ElerM/EUTplTjU08XNvWAQAmDM+KonIVuCJyzJmT2x7hpWdIu37f3WDc0FHkjQ71aa9AId1y2WjpFBPHYJL2KwO7rkLKqkcZvd0DLElDsMBWTz3kwG8FcBZQtjj54job0S0EsCZAN6fh6E6FH0vMPDceZ6UPeiTR7dFQhhHjSjh/edUIzTaSoHWXD149SfPDz8H9X/775fhsqXVMeQ3LpuGVZ88P7KuKSCXZUQuf41hUql2QZYROxJTmBDg2UdUyGRabNGMbO2t+uT5mM7NSQguxZGTR+J7bzs2dRtBR9Wk3I7lx1bur9kJ4ykODkC2aJk/Qf4c/S69Oekw3yfTxdMqMzB5b3TqmBF4fsu+SPlpY+JkNbazoh13cZ4xT0iygczO1mJMbuiUlJMNHooa+2detzBWRoYiJ2c8/JFzYoPBpgg6BT6xmIgrjpuGcxdMSN0GD/46ya4RUBlPkE34MkV4SZuU3YlIee0cHEQM6xmqAZbNHIv7Pngm3rhsGoBqwq254ztxub9tyugRYflJo9pidbT509dHcjm3VQOqtpLnQsmC26LnbkpqgU0LJo3MRLrHzBiL+z90Jt507DRlGSLKhdjrheCSuuXsHByGecpfHrLX/Olj23HVqbNx1uHj8eU7n8H6nQdw44WHS4k0kGL4JGQqilg0ZRQeenE7ujtbFCWiuPyYqVg0dRRKBQ9nf/FejCgVUEqZC6XgEe7/0JkY02HWtg55yC15IC+tPMzqmdEeB4dmQNOQO49q6t7KYOa8CV2hNyeTZIBqqtwZXCeh8gBvuPBwvHbx5Ji2rgIRhdP37/3gGehsLaKztYjf/eOpuOir95udFGdTo5ByXqiScjZarnYSjt4dHJpClhERSq+SBOaqqJSAWE6YPa66TcERpYIXxrDbYsa4Dozz4+oXSBaiTsJwn/wzXdIx5RXZF4SejhzRlD6Lg4MVmvIpkL2eB5q7Kvf7a46ejLZSAef54WY8GolQR2ZIbzzU+PnVJ+IwSRhfMEg4ZcyI2D4bfOrShbho0SRlkjMHh0MJw5cpNGhvqZwWPxj4gfPm4cjJI5V5Swoe4YKF8nwsM1KkEjbFr645KVxeLwlfWb4Yx3NvFsMNx8ny2qMy4Pzvb14qXVfWBiNaCjhz/vhMdTg4NAuaktxPnduNmy47KpLitbVYMEr5yqOtVMC/v3kpls1UhwsCwK3XnJRaWtCFIoqwtX844eJFk5ILOTg4GKMpyZ2IsPy46bnUxZPOD995vHRRiCUWBO3g4OBQDzQludcKp2SYGu/g4OBQTzhyHwb4/957Ch57Jb5AhYODg4MKjtyHAY6aOgpHTY3PcnVwcHBQoSnj3B0cHBwOdThyd3BwcGhCOHJ3cHBwaEI4cndwcHBoQjhyd3BwcGhCOHJ3cHBwaEI4cndwcHBoQjhyd3BwcGhCkLiW55AYQbQFwEsZqugGsDUnc2oBZ182NLp9QOPb6OzLjka0cQZjTJpOtSHIPSuIaAVjbNlQ26GCsy8bGt0+oPFtdPZlx3CwkYeTZRwcHByaEI7cHRwcHJoQzULuNw+1AQlw9mVDo9sHNL6Nzr7sGA42hmgKzd3BwcHBIYpm8dwdHBwcHDg4cndwcHBoQgxrcieiC4hoDRE9R0Q3DpEN3yWizUS0its2lojuJKJn/b9juH0f9u1dQ0Tn18G+aUR0NxE9RUSrieh9DWhjGxE9RERP+DZ+stFs9NssENFjRHR7o9lHRGuJ6G9E9DgRrWg0+/w2RxPRL4noaf9+PLFRbCSi+f61C/7tJqLrGsW+VGCMDct/AAoAngcwG0ALgCcALBgCO04DsBTAKm7b5wDc6H++EcC/+p8X+Ha2Apjl21+osX2TACz1P3cBeMa3o5FsJACd/ucSgAcBnNBINvrtXg/gxwBub8DfeS2AbmFbw9jnt/sDAO/yP7cAGN1oNvptFwC8CmBGI9pnfB5DbUCGH+BEAL/nvn8YwIeHyJaZiJL7GgCT/M+TAKyR2Qjg9wBOrLOtvwFwbqPaCKAdwKMAjm8kGwFMBXAXgLM4cm8k+2Tk3kj2jQTwIvwgjka0kWvrPAB/blT7TP8NZ1lmCoBXuO/r/G2NgAmMsY0A4P8d728fUpuJaCaAJah4xg1loy95PA5gM4A7GWONZuO/AfgQgDK3rZHsYwD+QESPENFVDWjfbABbAHzPl7b+k4g6GszGAMsB/MT/3Ij2GWE4kztJtjV6XOeQ2UxEnQBuAXAdY2y3rqhkW81tZIwNMsYWo+IhH0dECzXF62ojEV0CYDNj7BHTQyTban0NT2aMLQVwIYBrieg0TdmhsK+Iinz5TcbYEgD7UJE5VBiS+5CIWgC8FsAvkopKtjUU/wxncl8HYBr3fSqADUNki4hNRDQJAPy/m/3tQ2IzEZVQIfYfMcZ+1Yg2BmCM7QRwD4ALGsjGkwG8lojWAvgpgLOI6IcNZB8YYxv8v5sB3ArguEayz29znf9GBgC/RIXsG8lGoNI5PsoY2+R/bzT7jDGcyf1hAHOJaJbf2y4HcNsQ2xTgNgBX+p+vREXnDrYvJ6JWIpoFYC6Ah2ppCBERgO8AeIox9qUGtbGHiEb7n0cAOAfA041iI2Psw4yxqYyxmajcZ39kjL2lUewjog4i6go+o6IZr2oU+wCAMfYqgFeIaL6/6WwATzaSjT6uQFWSCexoJPvMMdSif8aBj4tQif54HsBHhsiGnwDYCKAfld78nQDGoTL49qz/dyxX/iO+vWsAXFgH+05B5XVxJYDH/X8XNZiNiwA85tu4CsD/9bc3jI1cu2egOqDaEPahomc/4f9bHTwLjWIf1+ZiACv83/nXAMY0ko2oDOZvAzCK29Yw9tn+c+kHHBwcHJoQw1mWcXBwcHBQwJG7g4ODQxPCkbuDg4NDE8KRu4ODg0MTwpG7g4ODQxPCkbuDg4NDE8KRu4ODg0MT4v8Hs3tn0K9Fw8YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > 195:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
